RowUtils { public static Object toRowValue(Object item, DataType type, Map<RowUtils.RowValueMetadata, Object> metadata) { String typeName = type.typeName(); switch (typeName) { case "binary": if (item instanceof ByteBuffer) { return ((ByteBuffer) item).array(); } else if (item instanceof byte[]) { return item; } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format", type)); } case "boolean": if (item instanceof Boolean) { return item; } String str = item.toString(); if ("true".equals(str.toLowerCase())) { return Boolean.TRUE; } else if ("false".equals(str.toLowerCase())) { return Boolean.FALSE; } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } case "date": if (item instanceof Long) { return new java.sql.Date((Long) item); } else if (item instanceof String) { return new java.sql.Date(DateTime.parse((String) item).getMillis()); } else if (item instanceof Date) { return new java.sql.Date(((Date) item).getTime()); } else if (item instanceof DateTime) { return new java.sql.Date(((DateTime) item).getMillis()); } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } case "timestamp": if (item instanceof Long) { return new Timestamp((Long) item); } else if (item instanceof String) { return new Timestamp(getDateTimeParser(metadata).parse((String) item).getMillis()); } else if (item instanceof Date) { return new Timestamp(((Date) item).getTime()); } else if (item instanceof DateTime) { return new Timestamp(((DateTime) item).getMillis()); } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } case "double": if (item instanceof Double) { return item; } else if (item instanceof Number) { return ((Number) item).doubleValue(); } else { try { return Double.parseDouble((String) item); } catch (Exception ex) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } } case "float": if (item instanceof Float) { return item; } else if (item instanceof Number) { return ((Number) item).floatValue(); } else { try { return Float.parseFloat((String) item); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } } case "integer": if (item instanceof Integer) { return item; } else if (item instanceof Number) { return ((Number) item).intValue(); } else { try { return Integer.parseInt((String) item); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } } case "long": if (item instanceof Long) { return item; } else if (item instanceof Number) { return ((Number) item).longValue(); } else { try { return Long.parseLong((String) item); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } } case "null": if (item == null) { return null; } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } case "byte": if (item instanceof Byte) { return item; } else if (item instanceof Number) { return ((Number) item).byteValue(); } else { try { return Byte.parseByte((String) item); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } } case "short": if (item instanceof Short) { return item; } else if (item instanceof Number) { return ((Number) item).shortValue(); } else { try { return Short.parseShort((String) item); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } } case "string": return item.toString(); case "array": DataType elementType = ((ArrayType) type).elementType(); ArrayList<Object> arrayList = new ArrayList<>(); if (item instanceof List) { for (Object value : (List<?>) item) { if (null != value) { try { value = toRowValue(value, elementType); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized element format: %s[%s]", type, elementType, value)); } } else if (!((ArrayType) type).containsNull()) { throw new RuntimeException(String.format("Type[%s] - Element cannot be 'null': %s[null]", type, elementType)); } arrayList.add(value); } return arrayList; } else if (item instanceof Row) { Row row = (Row) item; for (int i = 0; i < row.length(); i++) { Object value = row.get(i); if (null != value) { try { value = toRowValue(value, elementType); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized value format: %s[%s]", type, elementType, value)); } } else if (!((ArrayType) type).containsNull()) { throw new RuntimeException(String.format("Type[%s] - Value cannot be 'null': %s[null]", type, elementType)); } arrayList.add(value); } return arrayList; } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } case "map": MapType mapType = (MapType) type; DataType keyType = mapType.keyType(); DataType valueType = mapType.valueType(); HashMap<Object, Object> hashMap = new HashMap<>(); if (item instanceof Map) { Map<?, ?> map = (Map<?, ?>) item; for (Object k : map.keySet()) { Object key, value; try { key = toRowValue(k, keyType); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized key format: %s[%s]", type, keyType, k)); } value = map.get(k); if (null != value) { try { value = toRowValue(value, valueType); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized value format: %s[%s]", type, valueType, value)); } } else if (!mapType.valueContainsNull()) { throw new RuntimeException(String.format("Type[%s] - Value cannot be 'null': %s[null]", type, valueType)); } hashMap.put(key, value); } return hashMap; } else if (item instanceof Row) { Row row = (Row) item; StructType schema = row.schema(); if (null == schema) { throw new RuntimeException(String.format("Type[%s] - Invalid Row format, no schema found: Row[%s]", type, item)); } String[] fieldNames = schema.fieldNames(); for (int i = 0; i < fieldNames.length; i++) { Object value = row.get(i); if (null != value) { try { value = toRowValue(value, valueType); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized value format: %s[%s]", type, valueType, value)); } } else if (!mapType.valueContainsNull()) { throw new RuntimeException(String.format("Type[%s] - Value cannot be 'null': %s[null]", type, valueType)); } hashMap.put(fieldNames[i], value); } return hashMap; } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } case "struct": ArrayList<Object> valueList = new ArrayList<>(); if (item instanceof Map) { @SuppressWarnings("unchecked") Map<Object, Object> input = (Map<Object, Object>) item; for (StructField f : ((StructType) type).fields()) { if (input.containsKey(f.name())) { Object value = input.get(f.name()); if (null != value) { try { value = toRowValue(value, f.dataType()); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized value format: %s[%s]", type, f.dataType(), value)); } } else if (!f.nullable()) { throw new RuntimeException(String.format("Type[%s] - Value cannot be 'null': %s[null]", type, f.dataType())); } valueList.add(value); } else { throw new RuntimeException(String.format("Type[%s] - Key not found on input: %s[%s]", type, f.name(), f.dataType())); } } } else if (item instanceof List) { @SuppressWarnings("unchecked") List<Object> input = (List<Object>) item; StructField[] fields = ((StructType) type).fields(); if (fields.length != input.size()) { throw new RuntimeException(String.format("Type[%s] - Invalid size of input List: %s", type, item)); } for (int i = 0; i < fields.length; i++) { Object value = input.get(i); if (null != value) { try { value = toRowValue(value, fields[i].dataType()); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized element format: %s[%s]", type, fields[i].dataType(), value)); } } else if (!fields[i].nullable()) { throw new RuntimeException(String.format("Type[%s] - Element cannot be 'null': %s[null]", type, fields[i].dataType())); } valueList.add(value); } } else if (item instanceof Row) { Row input = (Row) item; StructField[] fields = ((StructType) type).fields(); if (fields.length != input.size()) { throw new RuntimeException(String.format("Type[%s] - Invalid size of input Row: %s", type, item)); } for (int i = 0; i < fields.length; i++) { Object value = input.get(i); if (null != value) { try { value = toRowValue(value, fields[i].dataType()); } catch (Exception e) { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized value format: %s[%s]", type, fields[i].dataType(), value)); } } else if (!fields[i].nullable()) { throw new RuntimeException(String.format("Type[%s] - Value cannot be 'null': %s[null]", type, fields[i].dataType())); } valueList.add(value); } } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } return RowFactory.create(valueList.toArray()); default: if (type.typeName().startsWith("decimal")) { if (item instanceof Long) { return BigDecimal.valueOf((Long) item, ((DecimalType) type).scale()); } else if (item instanceof Double) { return BigDecimal.valueOf((Double) item).setScale(((DecimalType) type).scale(), RoundingMode.HALF_DOWN); } else if (item instanceof String) { return new BigDecimal((String) item).setScale(((DecimalType) type).scale(), RoundingMode.HALF_DOWN); } else if (item instanceof BigDecimal) { return item; } else if (item instanceof BigInteger) { return new BigDecimal((BigInteger) item, ((DecimalType) type).scale()); } else { throw new RuntimeException(String.format("Type[%s] - Invalid or unrecognized input format: %s", type, item)); } } else { throw new RuntimeException(String.format("Type[%s] - StructField DataType unrecognized or not yet implemented", type)); } } }  static Object toRowValue(Object item, DataType type, Map<RowUtils.RowValueMetadata, Object> metadata); static Object toRowValue(Object item, DataType type); static Row subsetRow(Row row, StructType subsetSchema); static Row set(Row row, String fieldName, Object replacement); static Row append(Row row, Object value); static Row append(Row row, String fieldName, DataType fieldType, boolean nullable, Object value); static Row append(Row row, Row append); static Row remove(Row row, String fieldName); static Row remove(Row row, List<String> fieldNames); static Object[] valuesFor(Row row); static boolean different(Row first, Row second, List<String> valueFieldNames); static Column[] toColumnArray(List<String> columnList);  }
@Test public void testToRowValueBinary() { DataType field = DataTypes.BinaryType; byte[] byteArray = "Test".getBytes(); ByteBuffer byteBuffer = ByteBuffer.wrap(byteArray); assertEquals("Invalid byte[]", byteArray, RowUtils.toRowValue(byteArray, field)); assertEquals("Invalid ByteBuffer", byteArray, RowUtils.toRowValue(byteBuffer, field)); thrown.expect(RuntimeException.class); RowUtils.toRowValue(123, field); }
@Test public void testToRowValueBoolean() { DataType field = DataTypes.BooleanType; assertEquals("Invalid Boolean", true, RowUtils.toRowValue(true, field)); assertEquals("Invalid 'true'", true, RowUtils.toRowValue("true", field)); assertEquals("Invalid 'true'", true, RowUtils.toRowValue("TrUe", field)); assertEquals("Invalid 'false'", false, RowUtils.toRowValue("false", field)); assertEquals("Invalid 'false'", false, RowUtils.toRowValue("FaLsE", field)); try { RowUtils.toRowValue(123, field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } }
@Test public void testToRowValueCalendarInterval() { DataType field = DataTypes.CalendarIntervalType; thrown.expect(RuntimeException.class); thrown.expectMessage(CoreMatchers.containsString("StructField DataType unrecognized or not yet implemented")); RowUtils.toRowValue("INTERVAL 1 MONTH", field); }
@Test public void testToRowValueDate() { DataType field = DataTypes.DateType; DateTime dateObj = DateTime.parse("2017-01-01T00:00:00"); Date sqlDate = new Date(dateObj.getMillis()); assertEquals("Invalid Long", sqlDate, RowUtils.toRowValue(dateObj.getMillis(), field)); assertEquals("Invalid String", sqlDate, RowUtils.toRowValue("2017-001", field)); assertEquals("Invalid Date", sqlDate, RowUtils.toRowValue(dateObj.toDate(), field)); assertEquals("Invalid DateTime", sqlDate, RowUtils.toRowValue(dateObj, field)); thrown.expect(RuntimeException.class); thrown.expectMessage(CoreMatchers.containsString("Invalid or unrecognized input format")); RowUtils.toRowValue(123, field); }
@Test public void testToRowValueTimestamp() { DataType field = DataTypes.TimestampType; DateTime dateObj = DateTime.parse("2017-01-01T00:00:00"); Timestamp sqlTimestamp = new Timestamp(dateObj.getMillis()); assertEquals("Invalid Long", sqlTimestamp, RowUtils.toRowValue(dateObj.getMillis(), field)); assertEquals("Invalid String", sqlTimestamp, RowUtils.toRowValue("2017-001", field)); assertEquals("Invalid Date", sqlTimestamp, RowUtils.toRowValue(dateObj.toDate(), field)); assertEquals("Invalid DateTime", sqlTimestamp, RowUtils.toRowValue(dateObj, field)); Map<RowUtils.RowValueMetadata, Object> metadataNull = Maps.newHashMap(); Map<RowUtils.RowValueMetadata, Object> metadataEmpty = Maps.newHashMap(); Map<RowUtils.RowValueMetadata, Object> metadataFormat = Maps.newHashMap(); Set<String> empty = Sets.newHashSet(); Set<String> formats = Sets.newHashSet(); formats.add("yyyy-MM-dd HH:mm:ss.SSSSS"); metadataNull.put(RowUtils.RowValueMetadata.TIMESTAMP_FORMATS, null); metadataEmpty.put(RowUtils.RowValueMetadata.TIMESTAMP_FORMATS, empty); metadataFormat.put(RowUtils.RowValueMetadata.TIMESTAMP_FORMATS, formats); assertEquals("Invalid null metadata", sqlTimestamp, RowUtils.toRowValue("2017-01-01T00:00:00", field, null)); assertEquals("Invalid null format set in metadata", sqlTimestamp, RowUtils.toRowValue("2017-01-01T00:00:00", field, metadataNull)); assertEquals("Invalid format set", sqlTimestamp, RowUtils.toRowValue("2017-01-01 00:00:00.00000", field, metadataFormat)); assertEquals("Invalid empty format set", sqlTimestamp, RowUtils.toRowValue("2017-01-01T00:00:00", field, metadataEmpty)); thrown.expect(RuntimeException.class); thrown.expectMessage(CoreMatchers.containsString("Invalid or unrecognized input format")); RowUtils.toRowValue(123, field); }
@Test public void testToRowValueDouble() { DataType field = DataTypes.DoubleType; Double value = Double.valueOf("123"); assertEquals("Invalid Double", value, RowUtils.toRowValue(value, field)); assertEquals("Invalid Number", value, RowUtils.toRowValue(123L, field)); assertEquals("Invalid String", value, RowUtils.toRowValue("123", field)); try { RowUtils.toRowValue("foo", field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } try { RowUtils.toRowValue(ByteBuffer.allocate(1), field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } }
@Test public void testToRowValueFloat() { DataType field = DataTypes.FloatType; Float value = Float.valueOf("123"); assertEquals("Invalid Float", value, RowUtils.toRowValue(value, field)); assertEquals("Invalid Number", value, RowUtils.toRowValue(123L, field)); assertEquals("Invalid String", value, RowUtils.toRowValue("123", field)); try { RowUtils.toRowValue("foo", field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } try { RowUtils.toRowValue(ByteBuffer.allocate(1), field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } }
@Test public void testToRowValueInteger() { DataType field = DataTypes.IntegerType; Integer value = Integer.valueOf("123"); assertEquals("Invalid Integer", value, RowUtils.toRowValue(value, field)); assertEquals("Invalid Number", value, RowUtils.toRowValue(123L, field)); assertEquals("Invalid String", value, RowUtils.toRowValue("123", field)); try { RowUtils.toRowValue("foo", field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } try { RowUtils.toRowValue(ByteBuffer.allocate(1), field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } }
@Test public void testToRowValueLong() { DataType field = DataTypes.LongType; Long value = Long.valueOf("123"); assertEquals("Invalid Long", value, RowUtils.toRowValue(value, field)); assertEquals("Invalid Number", value, RowUtils.toRowValue(123, field)); assertEquals("Invalid String", value, RowUtils.toRowValue("123", field)); try { RowUtils.toRowValue("foo", field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } try { RowUtils.toRowValue(ByteBuffer.allocate(1), field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } }
@Test public void testToRowValueNull() { DataType field = DataTypes.NullType; assertEquals("Invalid NULL", null, RowUtils.toRowValue(null, field)); thrown.expect(RuntimeException.class); thrown.expectMessage("Invalid or unrecognized input format"); RowUtils.toRowValue(ByteBuffer.allocate(1), field); }
@Test public void testToRowValueByte() { DataType field = DataTypes.ByteType; Byte value = Byte.valueOf("123"); assertEquals("Invalid Byte", value, RowUtils.toRowValue(value, field)); assertEquals("Invalid Number", value, RowUtils.toRowValue(123, field)); assertEquals("Invalid String", value, RowUtils.toRowValue("123", field)); try { RowUtils.toRowValue("foo", field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } try { RowUtils.toRowValue(ByteBuffer.allocate(1), field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } }
@Test public void testToRowValueShort() { DataType field = DataTypes.ShortType; Short value = Short.valueOf("123"); assertEquals("Invalid Short", value, RowUtils.toRowValue(value, field)); assertEquals("Invalid Number", value, RowUtils.toRowValue(123, field)); assertEquals("Invalid String", value, RowUtils.toRowValue("123", field)); try { RowUtils.toRowValue("foo", field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } try { RowUtils.toRowValue(ByteBuffer.allocate(1), field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } }
@Test public void testToRowValueString() { DataType field = DataTypes.StringType; String value = "value"; assertEquals("Invalid String", value, RowUtils.toRowValue(value, field)); }
@Test public void testToRowValueDecimal() { DataType defaultField = DataTypes.createDecimalType(); BigDecimal defaultDecimal = new BigDecimal("10"); assertEquals("Invalid double", defaultDecimal, RowUtils.toRowValue(10.157D, defaultField)); assertEquals("Invalid string", defaultDecimal, RowUtils.toRowValue("10.157", defaultField)); assertEquals("Invalid BigDecimal", defaultDecimal, RowUtils.toRowValue(new BigDecimal("10"), defaultField)); assertEquals("Invalid long", defaultDecimal, RowUtils.toRowValue(10L, defaultField)); assertEquals("Invalid BigInteger", defaultDecimal, RowUtils.toRowValue(new BigInteger("10"), defaultField)); assertEquals("Invalid precision", 2, ((BigDecimal) RowUtils.toRowValue("10.157", defaultField)).precision()); assertEquals("Invalid scale", 0, ((BigDecimal) RowUtils.toRowValue("10.157", defaultField)).scale()); try { RowUtils.toRowValue(ByteBuffer.allocate(1), defaultField); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } DataType customField = DataTypes.createDecimalType(3, 2); BigDecimal customDecimal = new BigDecimal("1.23"); assertEquals("Invalid double", customDecimal, RowUtils.toRowValue(1.23D, customField)); assertEquals("Invalid string", customDecimal, RowUtils.toRowValue("1.23", customField)); assertEquals("Invalid BigDecimal", customDecimal, RowUtils.toRowValue(new BigDecimal("1.23"), customField)); assertEquals("Invalid long", customDecimal, RowUtils.toRowValue(123L, customField)); assertEquals("Invalid BigInteger", customDecimal, RowUtils.toRowValue(new BigInteger("123"), customField)); assertEquals("Invalid precision", 3, ((BigDecimal) RowUtils.toRowValue("1.23", customField)).precision()); assertEquals("Invalid scale", 2, ((BigDecimal) RowUtils.toRowValue("1.23", customField)).scale()); }
@Test public void testToRowValueArray() { DataType field = DataTypes.createArrayType(DataTypes.IntegerType); List<?> expectedInts = Lists.newArrayList(1, 2, 3); List<?> expectedNulls = Lists.newArrayList(1, null, 3); try { RowUtils.toRowValue(12.34, field); fail("Expected a RuntimeException for invalid type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } assertEquals("Invalid List of Ints", expectedInts, RowUtils.toRowValue(Lists.newArrayList(1, 2, 3), field)); assertEquals("Invalid List of Mixed", expectedInts, RowUtils.toRowValue(Lists.<Object>newArrayList("1", 2, 3L), field)); assertEquals("Invalid List of Explicit Nulls", expectedNulls, RowUtils.toRowValue(Lists.newArrayList(1, null, 3), field)); try { RowUtils.toRowValue(Lists.newArrayList(1, ByteBuffer.allocate(1), 3), field); fail("Expected a RuntimeException for invalid element conversion"); } catch (Exception e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized element format")); } assertEquals("Invalid List of Ints", expectedInts, RowUtils.toRowValue(RowFactory.create(1, 2, 3), field)); assertEquals("Invalid List of Mixed", expectedInts, RowUtils.toRowValue(RowFactory.create("1", 2, 3L), field)); assertEquals("Invalid List of Explicit Nulls", expectedNulls, RowUtils.toRowValue(RowFactory.create(1, null, 3), field)); try { RowUtils.toRowValue(RowFactory.create(1, ByteBuffer.allocate(1), 3), field); fail("Expected a RuntimeException for invalid element conversion"); } catch (Exception e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } }
@Test public void testToRowValueArrayNested() { DataType nested = DataTypes.createArrayType(DataTypes.createArrayType(DataTypes.IntegerType)); List<Integer> expectedInts = Lists.newArrayList(1, 2, 3); List<List<Integer>> expectedArrays = Lists.newArrayList(); expectedArrays.add(expectedInts); List<Integer> expectedInnerNulls = Lists.newArrayList(1, null, 3); List<List<Integer>> expectedOuterNulls = Lists.newArrayList(); expectedOuterNulls.add(expectedInnerNulls); List<List<Integer>> expectedNullArrays = Lists.newArrayList(); expectedNullArrays.add(null); List<Integer> testInts = Lists.newArrayList(1, 2, 3); List<List<Integer>> testArrayInts = Lists.newArrayList(); testArrayInts.add(testInts); assertEquals("Invalid List of Arrays", expectedArrays, RowUtils.toRowValue(testArrayInts, nested)); List<?> testMixed = Lists.<Object>newArrayList("1", 2, 3L); List<List<?>> testArrayMixed = Lists.newArrayList(); testArrayMixed.add(testMixed); assertEquals("Invalid List of Mixed", expectedArrays, RowUtils.toRowValue(testArrayInts, nested)); List<List<Integer>> testArrayNull = Lists.newArrayList(); testArrayNull.add(null); assertEquals("Invalid List of Null Arrays", expectedNullArrays, RowUtils.toRowValue(testArrayNull, nested)); try { RowUtils.toRowValue(Lists.newArrayList(ByteBuffer.allocate(1)), nested); fail("Expected a RuntimeException for invalid outer type conversion"); } catch (Exception e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized element format")); } List<Object> testInvalidElement = Lists.newArrayList(); testInvalidElement.add(1); testInvalidElement.add(false); testInvalidElement.add(3); List<List<Object>> testArrayInvalidElement = Lists.newArrayList(); testArrayInvalidElement.add(testInvalidElement); try { RowUtils.toRowValue(testArrayInvalidElement, nested); fail("Expected a RuntimeException for invalid inner type conversion"); } catch (Exception e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized element format")); } List<Object> testNullElement = Lists.newArrayList(); testNullElement.add(1); testNullElement.add(null); testNullElement.add(3); List<List<Object>> testArrayNullElement = Lists.newArrayList(); testArrayNullElement.add(testNullElement); assertEquals("Invalid List of Explicit Inner Nulls", expectedOuterNulls, RowUtils.toRowValue(testArrayNullElement, nested)); Row rowInts = RowFactory.create(RowFactory.create(1, 2, 3)); assertEquals("Invalid List of Arrays", expectedArrays, RowUtils.toRowValue(rowInts, nested)); Row rowMixed = RowFactory.create(RowFactory.create("1", 2, 3L)); assertEquals("Invalid List of Mixed", expectedArrays, RowUtils.toRowValue(rowMixed, nested)); assertEquals("Invalid List of Null Arrays", expectedNullArrays, RowUtils.toRowValue(testArrayNull, nested)); try { RowUtils.toRowValue(RowFactory.create(ByteBuffer.allocate(1)), nested); fail("Expected a RuntimeException for invalid outer type conversion"); } catch (Exception e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } try { RowUtils.toRowValue(RowFactory.create(RowFactory.create(1, false, 3)), nested); fail("Expected a RuntimeException for invalid inner type conversion"); } catch (Exception e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } Row validNull = RowFactory.create(RowFactory.create(1, null, 3)); assertEquals("Invalid List of Explicit Inner Nulls", expectedOuterNulls, RowUtils.toRowValue(validNull, nested)); List<Row> nestedRow = Lists.newArrayList(RowFactory.create(1, 2, 3)); assertEquals("Invalid List of Rows", expectedArrays, RowUtils.toRowValue(nestedRow, nested)); Row nestedList = RowFactory.create(Lists.newArrayList(1, 2, 3)); assertEquals("Invalid List of Lists", expectedArrays, RowUtils.toRowValue(nestedList, nested)); }
@Test public void testToRowValueMap() { DataType fieldNotNullable = DataTypes.createMapType(DataTypes.LongType, DataTypes.IntegerType, false); DataType fieldNullable = DataTypes.createMapType(DataTypes.LongType, DataTypes.IntegerType, true); Map<Object, Object> expectedValues = Maps.newHashMap(); expectedValues.put(9L, 1); expectedValues.put(8L, 2); Map<Object, Object> expectedNulls = Maps.newHashMap(); expectedNulls.put(9L, null); expectedNulls.put(8L, 2); Map<Object, Object> inputMap = Maps.newHashMap(); inputMap.put(9L, 1); inputMap.put(8L, 2); assertEquals("Invalid map of values", expectedValues, RowUtils.toRowValue(inputMap, fieldNotNullable)); inputMap.clear(); inputMap.put(9L, 1L); inputMap.put(8L, "2"); assertEquals("Invalid map of values", expectedValues, RowUtils.toRowValue(inputMap, fieldNotNullable)); inputMap.clear(); inputMap.put(9, 1); inputMap.put("8", 2); assertEquals("Invalid map of values", expectedValues, RowUtils.toRowValue(inputMap, fieldNotNullable)); try { RowUtils.toRowValue(ByteBuffer.allocate(1), fieldNotNullable); fail("Expected a RuntimeException for type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } try { inputMap.clear(); inputMap.put(9L, ByteBuffer.allocate(1)); inputMap.put(8L, 2); RowUtils.toRowValue(inputMap, fieldNotNullable); fail("Expected a RuntimeException for invalid value type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } try { inputMap.clear(); inputMap.put(ByteBuffer.allocate(1), 1); inputMap.put(8L, 2); RowUtils.toRowValue(inputMap, fieldNotNullable); fail("Expected a RuntimeException for invalid key type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized key format")); } try { inputMap.clear(); inputMap.put(null, 1); inputMap.put(8L, 2); RowUtils.toRowValue(inputMap, fieldNotNullable); fail("Expected a RuntimeException for null key"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized key format")); } inputMap.clear(); inputMap.put(9L, null); inputMap.put(8L, 2); assertEquals("Invalid 'null' value", expectedNulls, RowUtils.toRowValue(inputMap, fieldNullable)); }
@Test public void testToRowValueMapRow( final @Mocked Row inputRow, final @Mocked StructType rowSchema ) { DataType fieldNotNullable = DataTypes.createMapType(DataTypes.StringType, DataTypes.LongType, false); DataType fieldNullable = DataTypes.createMapType(DataTypes.StringType, DataTypes.LongType, true); Map<Object, Object> expectedValues = Maps.newHashMap(); expectedValues.put("field1", 1L); expectedValues.put("field2", 2L); Map<Object, Object> expectedNulls = Maps.newHashMap(); expectedNulls.put("field1", null); expectedNulls.put("field2", 2L); new Expectations() {{ inputRow.schema(); result = rowSchema; rowSchema.fieldNames(); result = new String[] {"field1", "field2"}; inputRow.get(0); returns(1L, 1L, ByteBuffer.allocate(1), null); inputRow.get(1); returns(2L, "2", 2L); }}; assertEquals("Invalid map of values", expectedValues, RowUtils.toRowValue(inputRow, fieldNotNullable)); assertEquals("Invalid map of values", expectedValues, RowUtils.toRowValue(inputRow, fieldNotNullable)); try { RowUtils.toRowValue(ByteBuffer.allocate(1), fieldNotNullable); fail("Expected a RuntimeException for type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized input format")); } try { RowUtils.toRowValue(inputRow, fieldNotNullable); fail("Expected a RuntimeException for invalid value type"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } try { RowUtils.toRowValue(inputRow, fieldNotNullable); fail("Expected a RuntimeException for 'null' value"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Value cannot be 'null'")); } assertEquals("Invalid null value", expectedNulls, RowUtils.toRowValue(inputRow, fieldNullable)); try { RowUtils.toRowValue(RowFactory.create(""), fieldNotNullable); fail("Expected a RuntimeException for a missing schema"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid Row format, no schema found")); } }
@Test public void testToRowValueMapNested() { DataType field = DataTypes.createMapType(DataTypes.StringType, DataTypes.createMapType(DataTypes.LongType, DataTypes.IntegerType, true) ); Map<Object, Object> expectedInnerMap = Maps.newHashMap(); expectedInnerMap.put(9L, 1); expectedInnerMap.put(8L, 2); Map<Object, Object> expectedOuterMap = Maps.newHashMap(); expectedOuterMap.put("outer", expectedInnerMap); Map<Object, Object> innerMap = Maps.newHashMap(); innerMap.put(9L, 1); innerMap.put(8L, 2); Map<Object, Object> outerMap = Maps.newHashMap(); outerMap.put("outer", innerMap); assertEquals("Invalid list of values", expectedOuterMap, RowUtils.toRowValue(outerMap, field)); }
@Test public void testToRowValueMapRowNested( final @Mocked Row inputRow, final @Mocked StructType innerSchema, final @Mocked StructType outerSchema ) { DataType field = DataTypes.createMapType(DataTypes.StringType, DataTypes.createMapType(DataTypes.StringType, DataTypes.IntegerType, true) ); Map<Object, Object> expectedInnerMap = Maps.newHashMap(); expectedInnerMap.put("field1", 1); expectedInnerMap.put("field2", 2); Map<Object, Object> expectedOuterMap = Maps.newHashMap(); expectedOuterMap.put("outer", expectedInnerMap); new Expectations() {{ inputRow.schema(); returns(outerSchema, innerSchema); outerSchema.fieldNames(); result = new String[] {"outer"}; innerSchema.fieldNames(); result = new String[] {"field1", "field2"}; inputRow.get(0); returns(inputRow, 1); inputRow.get(1); result = 2; }}; assertEquals("Invalid list of values", expectedOuterMap, RowUtils.toRowValue(inputRow, field)); }
@Test public void testToRowValueStruct() { DataType field = DataTypes.createStructType(Lists.newArrayList( DataTypes.createStructField("field1", DataTypes. LongType, true), DataTypes.createStructField("field2", DataTypes.IntegerType, false) )); Row expectedValues = RowFactory.create(9L, 2); Row expectedNulls = RowFactory.create(null, 2); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(Lists.<Object>newArrayList(9L, 2), field)); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(Lists.<Object>newArrayList("9", 2L), field)); try { RowUtils.toRowValue(Lists.newArrayList(9L), field); fail("Expected a RuntimeException for invalid length "); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid size of input List")); } try { RowUtils.toRowValue(Lists.<Object>newArrayList(9L, 2, 3), field); fail("Expected a RuntimeException for invalid length "); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid size of input List")); } try { RowUtils.toRowValue(Lists.newArrayList(ByteBuffer.allocate(1), 2), field); fail("Expected a RuntimeException for invalid type on nullable field"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized element format")); } try { RowUtils.toRowValue(Lists.newArrayList(9L, ByteBuffer.allocate(1)), field); fail("Expected a RuntimeException for invalid type on non-nullable field"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized element format")); } assertEquals("Invalid list of nulls", expectedNulls, RowUtils.toRowValue(Lists.newArrayList(null, 2), field)); try { RowUtils.toRowValue(Lists.newArrayList(9L, null), field); fail("Expected a RuntimeException for invalid nullable field"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Element cannot be 'null'")); } Map<Object, Object> inputMap = Maps.newHashMap(); inputMap.put("field1", 9L); inputMap.put("field2", 2); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(inputMap, field)); inputMap.clear(); inputMap.put("field1", "9"); inputMap.put("field2", 2L); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(inputMap, field)); try { inputMap.clear(); inputMap.put("field1".getBytes(), 9L); inputMap.put("field2", 2); RowUtils.toRowValue(inputMap, field); fail("Expected a RuntimeException for invalid or missing key"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Key not found on input")); } try { inputMap.clear(); inputMap.put("field1", "One"); inputMap.put("field2", 2); RowUtils.toRowValue(inputMap, field); fail("Expected a RuntimeException for invalid type on nullable field"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } try { inputMap.clear(); inputMap.put("field1", "One"); inputMap.put("field2", ByteBuffer.allocate(1)); RowUtils.toRowValue(inputMap, field); fail("Expected a RuntimeException for invalid type on non-nullable field"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } inputMap.clear(); inputMap.put("field1", null); inputMap.put("field2", 2); assertEquals("Invalid list of nulls", expectedNulls, RowUtils.toRowValue(inputMap, field)); try { inputMap.clear(); inputMap.put("field1", "One"); inputMap.put("field2", null); RowUtils.toRowValue(inputMap, field); fail("Expected a RuntimeException for invalid nullable field"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(RowFactory.create(9L, 2), field)); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(RowFactory.create("9", 2L), field)); try { RowUtils.toRowValue(RowFactory.create(9L), field); fail("Expected a RuntimeException for invalid length (lt)"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid size of input Row")); } try { RowUtils.toRowValue(RowFactory.create(9L, 2, 3), field); fail("Expected a RuntimeException for invalid length (gt)"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid size of input Row")); } try { RowUtils.toRowValue(RowFactory.create(ByteBuffer.allocate(1), 2), field); fail("Expected a RuntimeException for invalid type on nullable field"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } try { RowUtils.toRowValue(RowFactory.create(9L, ByteBuffer.allocate(1)), field); fail("Expected a RuntimeException for invalid type on non-nullable"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Invalid or unrecognized value format")); } assertEquals("Invalid list of nulls", expectedNulls, RowUtils.toRowValue(RowFactory.create(null, 2), field)); try { RowUtils.toRowValue(RowFactory.create(9L, null), field); fail("Expected a RuntimeException for invalid null on nullable field"); } catch (RuntimeException e) { assertThat(e.getMessage(), CoreMatchers.containsString("Value cannot be 'null'")); } }
@Test public void testToRowValueStructNested() { DataType field = DataTypes.createStructType(Lists.newArrayList( DataTypes.createStructField("outer", DataTypes.createStructType(Lists.newArrayList( DataTypes.createStructField("field1", DataTypes.LongType, true), DataTypes.createStructField("field2", DataTypes.IntegerType, false) )), true) )); Row expectedInnerValues = RowFactory.create(9L, 2); Row expectedValues = RowFactory.create(expectedInnerValues); Map<Object, Object> innerMap = Maps.newHashMap(); innerMap.put("field1", 9L); innerMap.put("field2", 2); List<?> innerList = Lists.<Object>newArrayList(9L, 2); Row innerRow = RowFactory.create(9L, 2); Map<Object, Object> outerMap = Maps.newHashMap(); outerMap.put("outer", innerMap); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerMap, field)); outerMap.put("outer", innerList); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerMap, field)); outerMap.put("outer", innerRow); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerMap, field)); List<Object> outerList = new ArrayList<>(); outerList.add(innerMap); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerList, field)); outerList.clear(); outerList.add(innerList); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerList, field)); outerList.clear(); outerList.add(innerRow); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerList, field)); Row outerRow = RowFactory.create(innerMap); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerRow, field)); outerRow = RowFactory.create(innerList); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerRow, field)); outerRow = RowFactory.create(innerRow); assertEquals("Invalid list of values", expectedValues, RowUtils.toRowValue(outerRow, field)); }