@Test public void testIndexWrite2() { Connector accCon = null; Instance inst; String[] args = new String[7]; args[0] = "src/test/resources/ResultsFile1.txt"; args[1] = "src/test/resources/testQuery2.txt"; args[2] = "instance"; args[3] = "mock"; args[4] = "user"; args[5] = "password"; args[6] = "table"; String query = null; try { query = FileUtils.readFileToString(new File(args[1])); } catch (IOException e1) { e1.printStackTrace(); } try { inst = new MockInstance(args[2]); accCon = inst.getConnector(args[4], args[5].getBytes()); if(accCon.tableOperations().exists(args[6])) { accCon.tableOperations().delete(args[6]); } accCon.tableOperations().create(args[6]); } catch (AccumuloException e) { e.printStackTrace(); } catch (AccumuloSecurityException e) { e.printStackTrace(); } catch (TableExistsException e) { e.printStackTrace(); } catch (TableNotFoundException e) { e.printStackTrace(); } int result = 5; try { result = ToolRunner.run(new IndexWritingTool(), args); } catch (Exception e) { e.printStackTrace(); } Assert.assertEquals(0, result); Scanner scan = null; try { scan = accCon.createScanner("table", new Authorizations()); } catch (TableNotFoundException e) { e.printStackTrace(); } scan.setRange(new Range()); int count = 0; for (Map.Entry<Key, Value> entry : scan) { String[] k = entry.getKey().getRow().toString().split("\u0000"); String[] c = entry.getKey().getColumnFamily().toString().split("\u0000"); if(count == 0) { Assert.assertEquals(k[0], "person1"); Assert.assertEquals(k[1], "person2"); Assert.assertEquals(k[2], "person3"); Assert.assertEquals(c[0],"x"); Assert.assertEquals(c[1],"y"); Assert.assertEquals(c[2],"z"); } else if(count == 2) { Assert.assertEquals(k[0], "person3"); Assert.assertEquals(k[1], "person4"); Assert.assertEquals(k[2], "person5"); Assert.assertEquals(c[0],"x"); Assert.assertEquals(c[1],"y"); Assert.assertEquals(c[2],"z"); } else if(count == 5) { Assert.assertEquals(k[0], "person6"); Assert.assertEquals(k[1], "person7"); Assert.assertEquals(k[2], "person8"); Assert.assertEquals(c[0],"x"); Assert.assertEquals(c[1],"y"); Assert.assertEquals(c[2],"z"); } count ++; System.out.println(count); } Assert.assertEquals(9, count); }
@Override public int run(final String[] args) throws Exception { Preconditions.checkArgument(args.length == 7, "java " + IndexWritingTool.class.getCanonicalName() + " hdfsSaveLocation sparqlFile cbinstance cbzk cbuser cbpassword rdfTablePrefix."); final String inputDir = PathUtils.clean(args[0]); final String sparqlFile = PathUtils.clean(args[1]); final String instStr = args[2]; final String zooStr = args[3]; final String userStr = args[4]; final String passStr = args[5]; final String tablePrefix = args[6]; final String sparql = FileUtils.readFileToString(new File(sparqlFile)); final Job job = new Job(getConf(), "Write HDFS Index to Accumulo"); job.setJarByClass(this.getClass()); final Configuration jobConf = job.getConfiguration(); jobConf.setBoolean("mapred.map.tasks.speculative.execution", false); setVarOrders(sparql, jobConf); TextInputFormat.setInputPaths(job, inputDir); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); job.setNumReduceTasks(0); String tableName; if (zooStr.equals("mock")) { tableName = tablePrefix; } else { tableName = tablePrefix + "INDEX_" + UUID.randomUUID().toString().replace("-", "").toUpperCase(); } setAccumuloOutput(instStr, zooStr, userStr, passStr, job, tableName); jobConf.set(sparql_key, sparql); final int complete = job.waitForCompletion(true) ? 0 : -1; if (complete == 0) { final String[] varOrders = jobConf.getStrings("varOrders"); final String orders = Joiner.on("\u0000").join(varOrders); Instance inst; if (zooStr.equals("mock")) { inst = new MockInstance(instStr); } else { inst = new ZooKeeperInstance(instStr, zooStr); } final Connector conn = inst.getConnector(userStr, passStr.getBytes(StandardCharsets.UTF_8)); final BatchWriter bw = conn.createBatchWriter(tableName, 10, 5000, 1); final Counters counters = job.getCounters(); final Counter c1 = counters.findCounter(cardCounter, cardCounter); final Mutation m = new Mutation("~SPARQL"); final Value v = new Value(sparql.getBytes(StandardCharsets.UTF_8)); m.put(new Text("" + c1.getValue()), new Text(orders), v); bw.addMutation(m); bw.close(); return complete; } else { return complete; } }
IndexWritingTool extends Configured implements Tool { @Override public int run(final String[] args) throws Exception { Preconditions.checkArgument(args.length == 7, "java " + IndexWritingTool.class.getCanonicalName() + " hdfsSaveLocation sparqlFile cbinstance cbzk cbuser cbpassword rdfTablePrefix."); final String inputDir = PathUtils.clean(args[0]); final String sparqlFile = PathUtils.clean(args[1]); final String instStr = args[2]; final String zooStr = args[3]; final String userStr = args[4]; final String passStr = args[5]; final String tablePrefix = args[6]; final String sparql = FileUtils.readFileToString(new File(sparqlFile)); final Job job = new Job(getConf(), "Write HDFS Index to Accumulo"); job.setJarByClass(this.getClass()); final Configuration jobConf = job.getConfiguration(); jobConf.setBoolean("mapred.map.tasks.speculative.execution", false); setVarOrders(sparql, jobConf); TextInputFormat.setInputPaths(job, inputDir); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); job.setNumReduceTasks(0); String tableName; if (zooStr.equals("mock")) { tableName = tablePrefix; } else { tableName = tablePrefix + "INDEX_" + UUID.randomUUID().toString().replace("-", "").toUpperCase(); } setAccumuloOutput(instStr, zooStr, userStr, passStr, job, tableName); jobConf.set(sparql_key, sparql); final int complete = job.waitForCompletion(true) ? 0 : -1; if (complete == 0) { final String[] varOrders = jobConf.getStrings("varOrders"); final String orders = Joiner.on("\u0000").join(varOrders); Instance inst; if (zooStr.equals("mock")) { inst = new MockInstance(instStr); } else { inst = new ZooKeeperInstance(instStr, zooStr); } final Connector conn = inst.getConnector(userStr, passStr.getBytes(StandardCharsets.UTF_8)); final BatchWriter bw = conn.createBatchWriter(tableName, 10, 5000, 1); final Counters counters = job.getCounters(); final Counter c1 = counters.findCounter(cardCounter, cardCounter); final Mutation m = new Mutation("~SPARQL"); final Value v = new Value(sparql.getBytes(StandardCharsets.UTF_8)); m.put(new Text("" + c1.getValue()), new Text(orders), v); bw.addMutation(m); bw.close(); return complete; } else { return complete; } } }
IndexWritingTool extends Configured implements Tool { @Override public int run(final String[] args) throws Exception { Preconditions.checkArgument(args.length == 7, "java " + IndexWritingTool.class.getCanonicalName() + " hdfsSaveLocation sparqlFile cbinstance cbzk cbuser cbpassword rdfTablePrefix."); final String inputDir = PathUtils.clean(args[0]); final String sparqlFile = PathUtils.clean(args[1]); final String instStr = args[2]; final String zooStr = args[3]; final String userStr = args[4]; final String passStr = args[5]; final String tablePrefix = args[6]; final String sparql = FileUtils.readFileToString(new File(sparqlFile)); final Job job = new Job(getConf(), "Write HDFS Index to Accumulo"); job.setJarByClass(this.getClass()); final Configuration jobConf = job.getConfiguration(); jobConf.setBoolean("mapred.map.tasks.speculative.execution", false); setVarOrders(sparql, jobConf); TextInputFormat.setInputPaths(job, inputDir); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); job.setNumReduceTasks(0); String tableName; if (zooStr.equals("mock")) { tableName = tablePrefix; } else { tableName = tablePrefix + "INDEX_" + UUID.randomUUID().toString().replace("-", "").toUpperCase(); } setAccumuloOutput(instStr, zooStr, userStr, passStr, job, tableName); jobConf.set(sparql_key, sparql); final int complete = job.waitForCompletion(true) ? 0 : -1; if (complete == 0) { final String[] varOrders = jobConf.getStrings("varOrders"); final String orders = Joiner.on("\u0000").join(varOrders); Instance inst; if (zooStr.equals("mock")) { inst = new MockInstance(instStr); } else { inst = new ZooKeeperInstance(instStr, zooStr); } final Connector conn = inst.getConnector(userStr, passStr.getBytes(StandardCharsets.UTF_8)); final BatchWriter bw = conn.createBatchWriter(tableName, 10, 5000, 1); final Counters counters = job.getCounters(); final Counter c1 = counters.findCounter(cardCounter, cardCounter); final Mutation m = new Mutation("~SPARQL"); final Value v = new Value(sparql.getBytes(StandardCharsets.UTF_8)); m.put(new Text("" + c1.getValue()), new Text(orders), v); bw.addMutation(m); bw.close(); return complete; } else { return complete; } }  }
IndexWritingTool extends Configured implements Tool { @Override public int run(final String[] args) throws Exception { Preconditions.checkArgument(args.length == 7, "java " + IndexWritingTool.class.getCanonicalName() + " hdfsSaveLocation sparqlFile cbinstance cbzk cbuser cbpassword rdfTablePrefix."); final String inputDir = PathUtils.clean(args[0]); final String sparqlFile = PathUtils.clean(args[1]); final String instStr = args[2]; final String zooStr = args[3]; final String userStr = args[4]; final String passStr = args[5]; final String tablePrefix = args[6]; final String sparql = FileUtils.readFileToString(new File(sparqlFile)); final Job job = new Job(getConf(), "Write HDFS Index to Accumulo"); job.setJarByClass(this.getClass()); final Configuration jobConf = job.getConfiguration(); jobConf.setBoolean("mapred.map.tasks.speculative.execution", false); setVarOrders(sparql, jobConf); TextInputFormat.setInputPaths(job, inputDir); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); job.setNumReduceTasks(0); String tableName; if (zooStr.equals("mock")) { tableName = tablePrefix; } else { tableName = tablePrefix + "INDEX_" + UUID.randomUUID().toString().replace("-", "").toUpperCase(); } setAccumuloOutput(instStr, zooStr, userStr, passStr, job, tableName); jobConf.set(sparql_key, sparql); final int complete = job.waitForCompletion(true) ? 0 : -1; if (complete == 0) { final String[] varOrders = jobConf.getStrings("varOrders"); final String orders = Joiner.on("\u0000").join(varOrders); Instance inst; if (zooStr.equals("mock")) { inst = new MockInstance(instStr); } else { inst = new ZooKeeperInstance(instStr, zooStr); } final Connector conn = inst.getConnector(userStr, passStr.getBytes(StandardCharsets.UTF_8)); final BatchWriter bw = conn.createBatchWriter(tableName, 10, 5000, 1); final Counters counters = job.getCounters(); final Counter c1 = counters.findCounter(cardCounter, cardCounter); final Mutation m = new Mutation("~SPARQL"); final Value v = new Value(sparql.getBytes(StandardCharsets.UTF_8)); m.put(new Text("" + c1.getValue()), new Text(orders), v); bw.addMutation(m); bw.close(); return complete; } else { return complete; } }  static void main(final String[] args); @Override int run(final String[] args); void setVarOrders(final String s, final Configuration conf); }
IndexWritingTool extends Configured implements Tool { @Override public int run(final String[] args) throws Exception { Preconditions.checkArgument(args.length == 7, "java " + IndexWritingTool.class.getCanonicalName() + " hdfsSaveLocation sparqlFile cbinstance cbzk cbuser cbpassword rdfTablePrefix."); final String inputDir = PathUtils.clean(args[0]); final String sparqlFile = PathUtils.clean(args[1]); final String instStr = args[2]; final String zooStr = args[3]; final String userStr = args[4]; final String passStr = args[5]; final String tablePrefix = args[6]; final String sparql = FileUtils.readFileToString(new File(sparqlFile)); final Job job = new Job(getConf(), "Write HDFS Index to Accumulo"); job.setJarByClass(this.getClass()); final Configuration jobConf = job.getConfiguration(); jobConf.setBoolean("mapred.map.tasks.speculative.execution", false); setVarOrders(sparql, jobConf); TextInputFormat.setInputPaths(job, inputDir); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); job.setNumReduceTasks(0); String tableName; if (zooStr.equals("mock")) { tableName = tablePrefix; } else { tableName = tablePrefix + "INDEX_" + UUID.randomUUID().toString().replace("-", "").toUpperCase(); } setAccumuloOutput(instStr, zooStr, userStr, passStr, job, tableName); jobConf.set(sparql_key, sparql); final int complete = job.waitForCompletion(true) ? 0 : -1; if (complete == 0) { final String[] varOrders = jobConf.getStrings("varOrders"); final String orders = Joiner.on("\u0000").join(varOrders); Instance inst; if (zooStr.equals("mock")) { inst = new MockInstance(instStr); } else { inst = new ZooKeeperInstance(instStr, zooStr); } final Connector conn = inst.getConnector(userStr, passStr.getBytes(StandardCharsets.UTF_8)); final BatchWriter bw = conn.createBatchWriter(tableName, 10, 5000, 1); final Counters counters = job.getCounters(); final Counter c1 = counters.findCounter(cardCounter, cardCounter); final Mutation m = new Mutation("~SPARQL"); final Value v = new Value(sparql.getBytes(StandardCharsets.UTF_8)); m.put(new Text("" + c1.getValue()), new Text(orders), v); bw.addMutation(m); bw.close(); return complete; } else { return complete; } }  static void main(final String[] args); @Override int run(final String[] args); void setVarOrders(final String s, final Configuration conf);  }