@Test public void requireThatUDFSupportsTensors() throws IOException { Schema schema = new Schema(); Tuple tuple = TupleFactory.getInstance().newTuple(); Map<String, Double> tensor = new HashMap<String, Double>() {{ put("x:label1,y:label2,z:label4", 2.0); put("x:label3", 3.0); }}; addToTuple("id", DataType.CHARARRAY, "123", schema, tuple); addToTuple("tensor", DataType.MAP, tensor, schema, tuple); VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "create-tensor-fields=tensor"); docOp.setInputSchema(schema); String json = docOp.exec(tuple); ObjectMapper m = new ObjectMapper(); JsonNode root = m.readTree(json); JsonNode fields = root.get("fields"); JsonNode tensorNode = fields.get("tensor"); JsonNode cells = tensorNode.get("cells"); assertEquals("label1", cells.get(0).get("address").get("x").asText()); assertEquals("label2", cells.get(0).get("address").get("y").asText()); assertEquals("label4", cells.get(0).get("address").get("z").asText()); assertEquals("label3", cells.get(1).get("address").get("x").asText()); assertEquals(2.0, cells.get(0).get("value").asDouble(), 1e-6); assertEquals(3.0, cells.get(1).get("value").asDouble(), 1e-6); }
@Override public String exec(Tuple tuple) throws IOException { if (tuple == null || tuple.size() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } return null; } if (template == null || template.length() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); return null; } if (operation == null) { warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2); return null; } String json = null; try { if (reporter != null) { reporter.progress(); } Schema inputSchema = getInputSchema(); Map<String, Object> fields = TupleTools.tupleMap(inputSchema, tuple); String docId = TupleTools.toString(fields, template); if (verbose) { System.out.println("Processing docId: "+ docId); } json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3); return null; } } catch (Exception e) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } StringBuilder sb = new StringBuilder(); sb.append("Caught exception processing input row: \n"); sb.append(tuple.toString()); sb.append("\nException: "); sb.append(getStackTraceAsString(e)); warnLog(sb.toString(), PigWarning.UDF_WARNING_4); return null; } if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation ok", 1); } return json; }
VespaDocumentOperation extends EvalFunc<String> { @Override public String exec(Tuple tuple) throws IOException { if (tuple == null || tuple.size() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } return null; } if (template == null || template.length() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); return null; } if (operation == null) { warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2); return null; } String json = null; try { if (reporter != null) { reporter.progress(); } Schema inputSchema = getInputSchema(); Map<String, Object> fields = TupleTools.tupleMap(inputSchema, tuple); String docId = TupleTools.toString(fields, template); if (verbose) { System.out.println("Processing docId: "+ docId); } json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3); return null; } } catch (Exception e) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } StringBuilder sb = new StringBuilder(); sb.append("Caught exception processing input row: \n"); sb.append(tuple.toString()); sb.append("\nException: "); sb.append(getStackTraceAsString(e)); warnLog(sb.toString(), PigWarning.UDF_WARNING_4); return null; } if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation ok", 1); } return json; } }
VespaDocumentOperation extends EvalFunc<String> { @Override public String exec(Tuple tuple) throws IOException { if (tuple == null || tuple.size() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } return null; } if (template == null || template.length() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); return null; } if (operation == null) { warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2); return null; } String json = null; try { if (reporter != null) { reporter.progress(); } Schema inputSchema = getInputSchema(); Map<String, Object> fields = TupleTools.tupleMap(inputSchema, tuple); String docId = TupleTools.toString(fields, template); if (verbose) { System.out.println("Processing docId: "+ docId); } json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3); return null; } } catch (Exception e) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } StringBuilder sb = new StringBuilder(); sb.append("Caught exception processing input row: \n"); sb.append(tuple.toString()); sb.append("\nException: "); sb.append(getStackTraceAsString(e)); warnLog(sb.toString(), PigWarning.UDF_WARNING_4); return null; } if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation ok", 1); } return json; } VespaDocumentOperation(String... params); }
VespaDocumentOperation extends EvalFunc<String> { @Override public String exec(Tuple tuple) throws IOException { if (tuple == null || tuple.size() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } return null; } if (template == null || template.length() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); return null; } if (operation == null) { warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2); return null; } String json = null; try { if (reporter != null) { reporter.progress(); } Schema inputSchema = getInputSchema(); Map<String, Object> fields = TupleTools.tupleMap(inputSchema, tuple); String docId = TupleTools.toString(fields, template); if (verbose) { System.out.println("Processing docId: "+ docId); } json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3); return null; } } catch (Exception e) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } StringBuilder sb = new StringBuilder(); sb.append("Caught exception processing input row: \n"); sb.append(tuple.toString()); sb.append("\nException: "); sb.append(getStackTraceAsString(e)); warnLog(sb.toString(), PigWarning.UDF_WARNING_4); return null; } if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation ok", 1); } return json; } VespaDocumentOperation(String... params); @Override String exec(Tuple tuple); static String create(Operation op, String docId, Map<String, Object> fields, Properties properties,
                                Schema schema); }
VespaDocumentOperation extends EvalFunc<String> { @Override public String exec(Tuple tuple) throws IOException { if (tuple == null || tuple.size() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } return null; } if (template == null || template.length() == 0) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1); return null; } if (operation == null) { warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2); return null; } String json = null; try { if (reporter != null) { reporter.progress(); } Schema inputSchema = getInputSchema(); Map<String, Object> fields = TupleTools.tupleMap(inputSchema, tuple); String docId = TupleTools.toString(fields, template); if (verbose) { System.out.println("Processing docId: "+ docId); } json = create(operation, docId, fields, properties, inputSchema); if (json == null || json.length() == 0) { warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3); return null; } } catch (Exception e) { if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1); } StringBuilder sb = new StringBuilder(); sb.append("Caught exception processing input row: \n"); sb.append(tuple.toString()); sb.append("\nException: "); sb.append(getStackTraceAsString(e)); warnLog(sb.toString(), PigWarning.UDF_WARNING_4); return null; } if (statusReporter != null) { statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation ok", 1); } return json; } VespaDocumentOperation(String... params); @Override String exec(Tuple tuple); static String create(Operation op, String docId, Map<String, Object> fields, Properties properties,
                                Schema schema);  }