@Test public void testQuanitization() throws Exception { String[] directives = new String[] { "parse-as-csv __col ,", "drop __col", "set columns id,first,last,dob,email,age,hrlywage,address,city,state,country,zip", "quantize hrlywage wagerange 0.0:20.0='LOW',21.0:75.0='MEDIUM',75.1:200.0='HIGH'", "set column wagerange (wagerange == null) ? \"NOT FOUND\" : wagerange" }; List<Row> rows = Arrays.asList( new Row("__col", "1098,Root,Joltie,01/26/1956,root@joltie.io,32,11.79,150 Mars Ave,Palo Alto,CA,USA,32826"), new Row("__col", "1091,Root,Joltie,01/26/1956,root1@joltie.io,32,129.13,150 Mars Ave,Palo Alto,CA,USA,32826"), new Row("__col", "1092,Root,Joltie,01/26/1956,root@mars.com,32,9.54,150 Mars Ave,Palo Alto,CA,USA,32826"), new Row("__col", "1093,Root,Joltie,01/26/1956,root@foo.com,32,7.89,150 Mars Ave,Palo Alto,CA,USA,32826"), new Row("__col", "1094,Root,Joltie,01/26/1956,windy@joltie.io,32,45.67,150 Mars Ave,Palo Alto,CA,USA,32826"), new Row("__col", "1094,Root,Joltie,01/26/1956,windy@joltie.io,32,20.7,150 Mars Ave,Palo Alto,CA,USA,32826") ); rows = TestingRig.execute(directives, rows); Assert.assertTrue(rows.size() == 6); int low = 0, medium = 0, high = 0, notfound = 0; for (Row row : rows) { String v = (String) row.getValue("wagerange"); if (v.equalsIgnoreCase("NOT FOUND")) { notfound++; } else if (v.equals("LOW")) { low++; } else if (v.equals("MEDIUM")) { medium++; } else if (v.equals("HIGH")) { high++; } } Assert.assertEquals(3, low); Assert.assertEquals(1, medium); Assert.assertEquals(1, high); Assert.assertEquals(1, notfound); }
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { List<Row> results = new ArrayList<>(); for (Row row : rows) { int idx = row.find(col1); if (idx != -1) { try { Object object = row.getValue(idx); if (object == null) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has null value. It should be a non-null 'String', " + "'Float' or 'Double'.", col1)); } Double d; if (object instanceof String) { d = Double.parseDouble((String) object); } else if (object instanceof Double) { d = (Double) object; } else if (object instanceof Float) { d = ((Float) object).doubleValue(); } else { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type '%s'. It should be of type 'String', " + "'Float' or 'Double'.", col1, object.getClass().getSimpleName())); } String value = rangeMap.get(d); int destIdx = row.find(col2); if (destIdx == -1) { row.add(col2, value); } else { row.setValue(destIdx, value); } } catch (NumberFormatException e) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type. It should be of type 'String', " + "'Float' or 'Double'.", col1), e); } } else { throw new DirectiveExecutionException(NAME, "Column '" + col1 + "' does not exist."); } results.add(row); } return results; }
Quantization implements Directive, Lineage { @Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { List<Row> results = new ArrayList<>(); for (Row row : rows) { int idx = row.find(col1); if (idx != -1) { try { Object object = row.getValue(idx); if (object == null) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has null value. It should be a non-null 'String', " + "'Float' or 'Double'.", col1)); } Double d; if (object instanceof String) { d = Double.parseDouble((String) object); } else if (object instanceof Double) { d = (Double) object; } else if (object instanceof Float) { d = ((Float) object).doubleValue(); } else { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type '%s'. It should be of type 'String', " + "'Float' or 'Double'.", col1, object.getClass().getSimpleName())); } String value = rangeMap.get(d); int destIdx = row.find(col2); if (destIdx == -1) { row.add(col2, value); } else { row.setValue(destIdx, value); } } catch (NumberFormatException e) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type. It should be of type 'String', " + "'Float' or 'Double'.", col1), e); } } else { throw new DirectiveExecutionException(NAME, "Column '" + col1 + "' does not exist."); } results.add(row); } return results; } }
Quantization implements Directive, Lineage { @Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { List<Row> results = new ArrayList<>(); for (Row row : rows) { int idx = row.find(col1); if (idx != -1) { try { Object object = row.getValue(idx); if (object == null) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has null value. It should be a non-null 'String', " + "'Float' or 'Double'.", col1)); } Double d; if (object instanceof String) { d = Double.parseDouble((String) object); } else if (object instanceof Double) { d = (Double) object; } else if (object instanceof Float) { d = ((Float) object).doubleValue(); } else { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type '%s'. It should be of type 'String', " + "'Float' or 'Double'.", col1, object.getClass().getSimpleName())); } String value = rangeMap.get(d); int destIdx = row.find(col2); if (destIdx == -1) { row.add(col2, value); } else { row.setValue(destIdx, value); } } catch (NumberFormatException e) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type. It should be of type 'String', " + "'Float' or 'Double'.", col1), e); } } else { throw new DirectiveExecutionException(NAME, "Column '" + col1 + "' does not exist."); } results.add(row); } return results; }  }
Quantization implements Directive, Lineage { @Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { List<Row> results = new ArrayList<>(); for (Row row : rows) { int idx = row.find(col1); if (idx != -1) { try { Object object = row.getValue(idx); if (object == null) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has null value. It should be a non-null 'String', " + "'Float' or 'Double'.", col1)); } Double d; if (object instanceof String) { d = Double.parseDouble((String) object); } else if (object instanceof Double) { d = (Double) object; } else if (object instanceof Float) { d = ((Float) object).doubleValue(); } else { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type '%s'. It should be of type 'String', " + "'Float' or 'Double'.", col1, object.getClass().getSimpleName())); } String value = rangeMap.get(d); int destIdx = row.find(col2); if (destIdx == -1) { row.add(col2, value); } else { row.setValue(destIdx, value); } } catch (NumberFormatException e) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type. It should be of type 'String', " + "'Float' or 'Double'.", col1), e); } } else { throw new DirectiveExecutionException(NAME, "Column '" + col1 + "' does not exist."); } results.add(row); } return results; }  @Override UsageDefinition define(); @Override void initialize(Arguments args); @Override void destroy(); @Override List<Row> execute(List<Row> rows, ExecutorContext context); @Override Mutation lineage(); }
Quantization implements Directive, Lineage { @Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { List<Row> results = new ArrayList<>(); for (Row row : rows) { int idx = row.find(col1); if (idx != -1) { try { Object object = row.getValue(idx); if (object == null) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has null value. It should be a non-null 'String', " + "'Float' or 'Double'.", col1)); } Double d; if (object instanceof String) { d = Double.parseDouble((String) object); } else if (object instanceof Double) { d = (Double) object; } else if (object instanceof Float) { d = ((Float) object).doubleValue(); } else { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type '%s'. It should be of type 'String', " + "'Float' or 'Double'.", col1, object.getClass().getSimpleName())); } String value = rangeMap.get(d); int destIdx = row.find(col2); if (destIdx == -1) { row.add(col2, value); } else { row.setValue(destIdx, value); } } catch (NumberFormatException e) { throw new DirectiveExecutionException( NAME, String.format("Column '%s' has invalid type. It should be of type 'String', " + "'Float' or 'Double'.", col1), e); } } else { throw new DirectiveExecutionException(NAME, "Column '" + col1 + "' does not exist."); } results.add(row); } return results; }  @Override UsageDefinition define(); @Override void initialize(Arguments args); @Override void destroy(); @Override List<Row> execute(List<Row> rows, ExecutorContext context); @Override Mutation lineage(); static final String NAME; }