VectorAssemblerMapper extends MISOMapper { @Override protected Object map(Object[] input) { if (null == input) { return null; } int pos = 0; Map<Integer, Double> map = new HashMap<>(); for (Object col : input) { if (null != col) { if (col instanceof Number) { map.put(pos++, ((Number)col).doubleValue()); } else if (col instanceof String) { Vector vec = VectorUtil.getVector(col); pos = appendVector(vec, map, pos); } else if (col instanceof Vector) { pos = appendVector((Vector) col, map, pos); } else { throw new UnsupportedOperationException("not support type of object."); } } else { switch (handleInvalid) { case ERROR: throw new NullPointerException("null value is found in vector assembler inputs."); case SKIP: return null; default: } } } Vector vec = new SparseVector(pos, map); if (map.size() * RATIO > pos) { vec = ((SparseVector)vec).toDenseVector(); } return vec; } VectorAssemblerMapper(TableSchema dataSchema, Params params);   }
@Test public void testToDense() throws Exception { TableSchema schema = new TableSchema(new String[]{"c0", "c1", "c2"}, new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING}); TableSchema outSchema = new TableSchema(new String[]{"c0", "c1", "c2", "out"}, new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING, VectorTypes.VECTOR}); Params params = new Params() .set(VectorAssemblerParams.SELECTED_COLS, new String[]{"c0", "c1", "c2"}) .set(VectorAssemblerParams.OUTPUT_COL, "out"); VectorAssemblerMapper mapper = new VectorAssemblerMapper(schema, params); assertEquals(mapper.map(Row.of(new DenseVector(new double[]{3.0, 4.0}), 3.0, new SparseVector(3, new int[]{0, 2}, new double[]{1.0, 4.0}))).getField(3), new DenseVector(new double[]{3.0, 4.0, 3.0, 1.0, 0.0, 4.0})); assertEquals(mapper.getOutputSchema(), outSchema); }
@Test public void testToSparse() throws Exception { TableSchema schema = new TableSchema(new String[]{"c0", "c1", "c2"}, new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING}); TableSchema outSchema = new TableSchema(new String[]{"c0", "out"}, new TypeInformation<?>[]{Types.STRING, VectorTypes.VECTOR}); Params params = new Params() .set(VectorAssemblerParams.SELECTED_COLS, new String[]{"c0", "c1", "c2"}) .set(VectorAssemblerParams.OUTPUT_COL, "out") .set(VectorAssemblerParams.RESERVED_COLS, new String[]{"c0"}); VectorAssemblerMapper mapper = new VectorAssemblerMapper(schema, params); assertEquals(mapper.map(Row.of(new DenseVector(new double[]{3.0, 4.0}), 3.0, new SparseVector(11, new int[]{0, 10}, new double[]{1.0, 4.0}))).getField(1), new SparseVector(14, new int[]{0, 1, 2, 3, 13}, new double[]{3.0, 4.0, 3.0, 1.0, 4.0})); assertEquals(mapper.getOutputSchema(), outSchema); }
@Test public void testSkip() throws Exception { TableSchema schema = new TableSchema(new String[]{"c0", "c1", "c2"}, new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING}); TableSchema outSchema = new TableSchema(new String[]{"c0", "out"}, new TypeInformation<?>[]{Types.STRING, VectorTypes.VECTOR}); Params params = new Params() .set(VectorAssemblerParams.SELECTED_COLS, new String[]{"c0", "c1", "c2"}) .set(VectorAssemblerParams.OUTPUT_COL, "out") .set(VectorAssemblerParams.HANDLE_INVALID, HandleInvalidMethod.SKIP) .set(VectorAssemblerParams.RESERVED_COLS, new String[]{"c0"}); VectorAssemblerMapper mapper = new VectorAssemblerMapper(schema, params); assertEquals(mapper.map(Row.of(new DenseVector(new double[]{3.0, 4.0}), null, new SparseVector(11, new int[]{0, 10}, new double[]{1.0, 4.0}))).getField(1), null); assertEquals(mapper.getOutputSchema(), outSchema); }
@Test public void testStringValue() throws Exception { TableSchema schema = new TableSchema(new String[]{"c0", "c1", "c2"}, new TypeInformation<?>[]{Types.STRING, Types.DOUBLE, Types.STRING}); TableSchema outSchema = new TableSchema(new String[]{"c0", "out"}, new TypeInformation<?>[]{Types.STRING, VectorTypes.VECTOR}); Params params = new Params() .set(VectorAssemblerParams.SELECTED_COLS, new String[]{"c0", "c1", "c2"}) .set(VectorAssemblerParams.OUTPUT_COL, "out") .set(VectorAssemblerParams.HANDLE_INVALID, HandleInvalidMethod.SKIP) .set(VectorAssemblerParams.RESERVED_COLS, new String[]{"c0"}); VectorAssemblerMapper mapper = new VectorAssemblerMapper(schema, params); assertEquals(mapper.map(Row.of(new DenseVector(new double[]{3.0, 4.0}), "1", new SparseVector(11, new int[]{0, 10}, new double[]{1.0, 4.0}))).getField(1), new SparseVector(14, new int[]{0, 1, 2, 3, 13}, new double[]{3.0, 4.0, 1.0, 1.0, 4.0})); assertEquals(mapper.getOutputSchema(), outSchema); }