@Test public void testConvertToAvroNull() { Assert.assertNull("Avro null", JsonUtil.convertToAvro(GenericData.get(), null, Schema.create(Schema.Type.NULL))); Assert.assertNull("Avro nullable long", JsonUtil.convertToAvro(GenericData.get(), null, SchemaBuilder.nullable().longType())); Assert.assertNull("Avro long", JsonUtil.convertToAvro(GenericData.get(), null, SchemaBuilder.builder().longType())); }
public static Object convertToAvro(GenericData model, JsonNode datum, Schema schema) { if (datum == null) { return null; } switch (schema.getType()) { case RECORD: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to record: %s", datum); Object record = model.newRecord(null, schema); for (Schema.Field field : schema.getFields()) { model.setField(record, field.name(), field.pos(), convertField(model, datum.get(field.name()), field)); } return record; case MAP: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to map: %s", datum); Map<String, Object> map = Maps.newLinkedHashMap(); Iterator<Map.Entry<String, JsonNode>> iter = datum.fields(); while (iter.hasNext()) { Map.Entry<String, JsonNode> entry = iter.next(); map.put(entry.getKey(), convertToAvro( model, entry.getValue(), schema.getValueType())); } return map; case ARRAY: DatasetRecordException.check(datum.isArray(), "Cannot convert to array: %s", datum); List<Object> list = Lists.newArrayListWithExpectedSize(datum.size()); for (JsonNode element : datum) { list.add(convertToAvro(model, element, schema.getElementType())); } return list; case UNION: return convertToAvro(model, datum, resolveUnion(datum, schema.getTypes())); case BOOLEAN: DatasetRecordException.check(datum.isBoolean(), "Cannot convert to boolean: %s", datum); return datum.booleanValue(); case FLOAT: DatasetRecordException.check(datum.isFloat() || datum.isInt(), "Cannot convert to float: %s", datum); return datum.floatValue(); case DOUBLE: DatasetRecordException.check( datum.isDouble() || datum.isFloat() || datum.isLong() || datum.isInt(), "Cannot convert to double: %s", datum); return datum.doubleValue(); case INT: DatasetRecordException.check(datum.isInt(), "Cannot convert to int: %s", datum); return datum.intValue(); case LONG: DatasetRecordException.check(datum.isLong() || datum.isInt(), "Cannot convert to long: %s", datum); return datum.longValue(); case STRING: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return datum.textValue(); case ENUM: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return model.createEnum(datum.textValue(), schema); case BYTES: DatasetRecordException.check(datum.isBinary(), "Cannot convert to binary: %s", datum); try { return ByteBuffer.wrap(datum.binaryValue()); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } case FIXED: DatasetRecordException.check(datum.isBinary(), "Cannot convert to fixed: %s", datum); byte[] bytes; try { bytes = datum.binaryValue(); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } DatasetRecordException.check(bytes.length < schema.getFixedSize(), "Binary data is too short: %s bytes for %s", bytes.length, schema); return model.createFixed(null, bytes, schema); case NULL: return null; default: throw new IllegalArgumentException("Unknown schema type: " + schema); } }
JsonUtil { public static Object convertToAvro(GenericData model, JsonNode datum, Schema schema) { if (datum == null) { return null; } switch (schema.getType()) { case RECORD: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to record: %s", datum); Object record = model.newRecord(null, schema); for (Schema.Field field : schema.getFields()) { model.setField(record, field.name(), field.pos(), convertField(model, datum.get(field.name()), field)); } return record; case MAP: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to map: %s", datum); Map<String, Object> map = Maps.newLinkedHashMap(); Iterator<Map.Entry<String, JsonNode>> iter = datum.fields(); while (iter.hasNext()) { Map.Entry<String, JsonNode> entry = iter.next(); map.put(entry.getKey(), convertToAvro( model, entry.getValue(), schema.getValueType())); } return map; case ARRAY: DatasetRecordException.check(datum.isArray(), "Cannot convert to array: %s", datum); List<Object> list = Lists.newArrayListWithExpectedSize(datum.size()); for (JsonNode element : datum) { list.add(convertToAvro(model, element, schema.getElementType())); } return list; case UNION: return convertToAvro(model, datum, resolveUnion(datum, schema.getTypes())); case BOOLEAN: DatasetRecordException.check(datum.isBoolean(), "Cannot convert to boolean: %s", datum); return datum.booleanValue(); case FLOAT: DatasetRecordException.check(datum.isFloat() || datum.isInt(), "Cannot convert to float: %s", datum); return datum.floatValue(); case DOUBLE: DatasetRecordException.check( datum.isDouble() || datum.isFloat() || datum.isLong() || datum.isInt(), "Cannot convert to double: %s", datum); return datum.doubleValue(); case INT: DatasetRecordException.check(datum.isInt(), "Cannot convert to int: %s", datum); return datum.intValue(); case LONG: DatasetRecordException.check(datum.isLong() || datum.isInt(), "Cannot convert to long: %s", datum); return datum.longValue(); case STRING: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return datum.textValue(); case ENUM: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return model.createEnum(datum.textValue(), schema); case BYTES: DatasetRecordException.check(datum.isBinary(), "Cannot convert to binary: %s", datum); try { return ByteBuffer.wrap(datum.binaryValue()); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } case FIXED: DatasetRecordException.check(datum.isBinary(), "Cannot convert to fixed: %s", datum); byte[] bytes; try { bytes = datum.binaryValue(); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } DatasetRecordException.check(bytes.length < schema.getFixedSize(), "Binary data is too short: %s bytes for %s", bytes.length, schema); return model.createFixed(null, bytes, schema); case NULL: return null; default: throw new IllegalArgumentException("Unknown schema type: " + schema); } } }
JsonUtil { public static Object convertToAvro(GenericData model, JsonNode datum, Schema schema) { if (datum == null) { return null; } switch (schema.getType()) { case RECORD: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to record: %s", datum); Object record = model.newRecord(null, schema); for (Schema.Field field : schema.getFields()) { model.setField(record, field.name(), field.pos(), convertField(model, datum.get(field.name()), field)); } return record; case MAP: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to map: %s", datum); Map<String, Object> map = Maps.newLinkedHashMap(); Iterator<Map.Entry<String, JsonNode>> iter = datum.fields(); while (iter.hasNext()) { Map.Entry<String, JsonNode> entry = iter.next(); map.put(entry.getKey(), convertToAvro( model, entry.getValue(), schema.getValueType())); } return map; case ARRAY: DatasetRecordException.check(datum.isArray(), "Cannot convert to array: %s", datum); List<Object> list = Lists.newArrayListWithExpectedSize(datum.size()); for (JsonNode element : datum) { list.add(convertToAvro(model, element, schema.getElementType())); } return list; case UNION: return convertToAvro(model, datum, resolveUnion(datum, schema.getTypes())); case BOOLEAN: DatasetRecordException.check(datum.isBoolean(), "Cannot convert to boolean: %s", datum); return datum.booleanValue(); case FLOAT: DatasetRecordException.check(datum.isFloat() || datum.isInt(), "Cannot convert to float: %s", datum); return datum.floatValue(); case DOUBLE: DatasetRecordException.check( datum.isDouble() || datum.isFloat() || datum.isLong() || datum.isInt(), "Cannot convert to double: %s", datum); return datum.doubleValue(); case INT: DatasetRecordException.check(datum.isInt(), "Cannot convert to int: %s", datum); return datum.intValue(); case LONG: DatasetRecordException.check(datum.isLong() || datum.isInt(), "Cannot convert to long: %s", datum); return datum.longValue(); case STRING: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return datum.textValue(); case ENUM: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return model.createEnum(datum.textValue(), schema); case BYTES: DatasetRecordException.check(datum.isBinary(), "Cannot convert to binary: %s", datum); try { return ByteBuffer.wrap(datum.binaryValue()); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } case FIXED: DatasetRecordException.check(datum.isBinary(), "Cannot convert to fixed: %s", datum); byte[] bytes; try { bytes = datum.binaryValue(); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } DatasetRecordException.check(bytes.length < schema.getFixedSize(), "Binary data is too short: %s bytes for %s", bytes.length, schema); return model.createFixed(null, bytes, schema); case NULL: return null; default: throw new IllegalArgumentException("Unknown schema type: " + schema); } }  }
JsonUtil { public static Object convertToAvro(GenericData model, JsonNode datum, Schema schema) { if (datum == null) { return null; } switch (schema.getType()) { case RECORD: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to record: %s", datum); Object record = model.newRecord(null, schema); for (Schema.Field field : schema.getFields()) { model.setField(record, field.name(), field.pos(), convertField(model, datum.get(field.name()), field)); } return record; case MAP: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to map: %s", datum); Map<String, Object> map = Maps.newLinkedHashMap(); Iterator<Map.Entry<String, JsonNode>> iter = datum.fields(); while (iter.hasNext()) { Map.Entry<String, JsonNode> entry = iter.next(); map.put(entry.getKey(), convertToAvro( model, entry.getValue(), schema.getValueType())); } return map; case ARRAY: DatasetRecordException.check(datum.isArray(), "Cannot convert to array: %s", datum); List<Object> list = Lists.newArrayListWithExpectedSize(datum.size()); for (JsonNode element : datum) { list.add(convertToAvro(model, element, schema.getElementType())); } return list; case UNION: return convertToAvro(model, datum, resolveUnion(datum, schema.getTypes())); case BOOLEAN: DatasetRecordException.check(datum.isBoolean(), "Cannot convert to boolean: %s", datum); return datum.booleanValue(); case FLOAT: DatasetRecordException.check(datum.isFloat() || datum.isInt(), "Cannot convert to float: %s", datum); return datum.floatValue(); case DOUBLE: DatasetRecordException.check( datum.isDouble() || datum.isFloat() || datum.isLong() || datum.isInt(), "Cannot convert to double: %s", datum); return datum.doubleValue(); case INT: DatasetRecordException.check(datum.isInt(), "Cannot convert to int: %s", datum); return datum.intValue(); case LONG: DatasetRecordException.check(datum.isLong() || datum.isInt(), "Cannot convert to long: %s", datum); return datum.longValue(); case STRING: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return datum.textValue(); case ENUM: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return model.createEnum(datum.textValue(), schema); case BYTES: DatasetRecordException.check(datum.isBinary(), "Cannot convert to binary: %s", datum); try { return ByteBuffer.wrap(datum.binaryValue()); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } case FIXED: DatasetRecordException.check(datum.isBinary(), "Cannot convert to fixed: %s", datum); byte[] bytes; try { bytes = datum.binaryValue(); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } DatasetRecordException.check(bytes.length < schema.getFixedSize(), "Binary data is too short: %s bytes for %s", bytes.length, schema); return model.createFixed(null, bytes, schema); case NULL: return null; default: throw new IllegalArgumentException("Unknown schema type: " + schema); } }  static Iterator<JsonNode> parser(final InputStream stream); static JsonNode parse(String json); static T parse(String json, Class<T> returnType); static JsonNode parse(File file); static T parse(File file, Class<T> returnType); static JsonNode parse(InputStream in); static T parse(InputStream in, Class<T> returnType); @edu.umd.cs.findbugs.annotations.SuppressWarnings( value="BC_UNCONFIRMED_CAST", justification="Uses precondition to validate casts") static T visit(JsonNode node, JsonTreeVisitor<T> visitor); static Object convertToAvro(GenericData model, JsonNode datum,
                                     Schema schema); static Schema inferSchema(InputStream incoming, final String name,
                                   int numRecords); static Schema inferSchema(JsonNode node, String name); static Schema inferSchemaWithMaps(JsonNode node, String name); }
JsonUtil { public static Object convertToAvro(GenericData model, JsonNode datum, Schema schema) { if (datum == null) { return null; } switch (schema.getType()) { case RECORD: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to record: %s", datum); Object record = model.newRecord(null, schema); for (Schema.Field field : schema.getFields()) { model.setField(record, field.name(), field.pos(), convertField(model, datum.get(field.name()), field)); } return record; case MAP: DatasetRecordException.check(datum.isObject(), "Cannot convert non-object to map: %s", datum); Map<String, Object> map = Maps.newLinkedHashMap(); Iterator<Map.Entry<String, JsonNode>> iter = datum.fields(); while (iter.hasNext()) { Map.Entry<String, JsonNode> entry = iter.next(); map.put(entry.getKey(), convertToAvro( model, entry.getValue(), schema.getValueType())); } return map; case ARRAY: DatasetRecordException.check(datum.isArray(), "Cannot convert to array: %s", datum); List<Object> list = Lists.newArrayListWithExpectedSize(datum.size()); for (JsonNode element : datum) { list.add(convertToAvro(model, element, schema.getElementType())); } return list; case UNION: return convertToAvro(model, datum, resolveUnion(datum, schema.getTypes())); case BOOLEAN: DatasetRecordException.check(datum.isBoolean(), "Cannot convert to boolean: %s", datum); return datum.booleanValue(); case FLOAT: DatasetRecordException.check(datum.isFloat() || datum.isInt(), "Cannot convert to float: %s", datum); return datum.floatValue(); case DOUBLE: DatasetRecordException.check( datum.isDouble() || datum.isFloat() || datum.isLong() || datum.isInt(), "Cannot convert to double: %s", datum); return datum.doubleValue(); case INT: DatasetRecordException.check(datum.isInt(), "Cannot convert to int: %s", datum); return datum.intValue(); case LONG: DatasetRecordException.check(datum.isLong() || datum.isInt(), "Cannot convert to long: %s", datum); return datum.longValue(); case STRING: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return datum.textValue(); case ENUM: DatasetRecordException.check(datum.isTextual(), "Cannot convert to string: %s", datum); return model.createEnum(datum.textValue(), schema); case BYTES: DatasetRecordException.check(datum.isBinary(), "Cannot convert to binary: %s", datum); try { return ByteBuffer.wrap(datum.binaryValue()); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } case FIXED: DatasetRecordException.check(datum.isBinary(), "Cannot convert to fixed: %s", datum); byte[] bytes; try { bytes = datum.binaryValue(); } catch (IOException e) { throw new DatasetRecordException("Failed to read JSON binary", e); } DatasetRecordException.check(bytes.length < schema.getFixedSize(), "Binary data is too short: %s bytes for %s", bytes.length, schema); return model.createFixed(null, bytes, schema); case NULL: return null; default: throw new IllegalArgumentException("Unknown schema type: " + schema); } }  static Iterator<JsonNode> parser(final InputStream stream); static JsonNode parse(String json); static T parse(String json, Class<T> returnType); static JsonNode parse(File file); static T parse(File file, Class<T> returnType); static JsonNode parse(InputStream in); static T parse(InputStream in, Class<T> returnType); @edu.umd.cs.findbugs.annotations.SuppressWarnings( value="BC_UNCONFIRMED_CAST", justification="Uses precondition to validate casts") static T visit(JsonNode node, JsonTreeVisitor<T> visitor); static Object convertToAvro(GenericData model, JsonNode datum,
                                     Schema schema); static Schema inferSchema(InputStream incoming, final String name,
                                   int numRecords); static Schema inferSchema(JsonNode node, String name); static Schema inferSchemaWithMaps(JsonNode node, String name);  }