@Test public void testReadRawWithDifferentFieldName() throws IOException, MalformedRecordException { final List<RecordField> fields = getDefaultFields(); final RecordSchema schema = new SimpleRecordSchema(fields); final String headerLine = "id, name, balance, address, city, state, zipCode, continent"; final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111, North America"; final String csvData = headerLine + "\n" + inputRecord; final byte[] inputData = csvData.getBytes(); try (final InputStream bais = new ByteArrayInputStream(inputData); final JacksonCSVRecordReader reader = createReader(bais, schema, format)) { final Record record = reader.nextRecord(); assertNotNull(record); assertEquals("1", record.getValue("id")); assertEquals("John", record.getValue("name")); assertEquals("40.80", record.getValue("balance")); assertEquals("123 My Street", record.getValue("address")); assertEquals("My City", record.getValue("city")); assertEquals("MS", record.getValue("state")); assertEquals("11111", record.getValue("zipCode")); assertNull(record.getValue("country")); assertNull(record.getValue("continent")); assertNull(reader.nextRecord()); } try (final InputStream bais = new ByteArrayInputStream(inputData); final JacksonCSVRecordReader reader = createReader(bais, schema, format)) { final Record record = reader.nextRecord(false, false); assertNotNull(record); assertEquals("1", record.getValue("id")); assertEquals("John", record.getValue("name")); assertEquals("40.80", record.getValue("balance")); assertEquals("123 My Street", record.getValue("address")); assertEquals("My City", record.getValue("city")); assertEquals("MS", record.getValue("state")); assertEquals("11111", record.getValue("zipCode")); assertNull(record.getValue("country")); assertEquals("North America", record.getValue("continent")); assertNull(reader.nextRecord(false, false)); } }
@Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); if (recordStream.hasNext()) { String[] csvRecord = recordStream.next(); if (rawFieldNames == null) { if (!hasHeader || ignoreHeader) { rawFieldNames = schema.getFieldNames(); } else { rawFieldNames = Arrays.asList(csvRecord); if (recordStream.hasNext()) { csvRecord = recordStream.next(); } else { return null; } } } boolean foundRecord = true; if (csvRecord == null || (csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = false; while (recordStream.hasNext()) { csvRecord = recordStream.next(); if (csvRecord != null && !(csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = true; break; } } } if (!foundRecord) { return null; } final Map<String, Object> values = new HashMap<>(rawFieldNames.size() * 2); final int numFieldNames = rawFieldNames.size(); for (int i = 0; i < csvRecord.length; i++) { final String rawFieldName = numFieldNames <= i ? "unknown_field_index_" + i : rawFieldNames.get(i); String rawValue = (i >= csvRecord.length) ? null : csvRecord[i]; final Optional<DataType> dataTypeOption = schema.getDataType(rawFieldName); if (!dataTypeOption.isPresent() && dropUnknownFields) { continue; } final Object value; if (coerceTypes && dataTypeOption.isPresent()) { value = convert(rawValue, dataTypeOption.get(), rawFieldName); } else if (dataTypeOption.isPresent()) { value = convertSimpleIfPossible(rawValue, dataTypeOption.get(), rawFieldName); } else { value = rawValue; } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; }
JacksonCSVRecordReader extends AbstractCSVRecordReader { @Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); if (recordStream.hasNext()) { String[] csvRecord = recordStream.next(); if (rawFieldNames == null) { if (!hasHeader || ignoreHeader) { rawFieldNames = schema.getFieldNames(); } else { rawFieldNames = Arrays.asList(csvRecord); if (recordStream.hasNext()) { csvRecord = recordStream.next(); } else { return null; } } } boolean foundRecord = true; if (csvRecord == null || (csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = false; while (recordStream.hasNext()) { csvRecord = recordStream.next(); if (csvRecord != null && !(csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = true; break; } } } if (!foundRecord) { return null; } final Map<String, Object> values = new HashMap<>(rawFieldNames.size() * 2); final int numFieldNames = rawFieldNames.size(); for (int i = 0; i < csvRecord.length; i++) { final String rawFieldName = numFieldNames <= i ? "unknown_field_index_" + i : rawFieldNames.get(i); String rawValue = (i >= csvRecord.length) ? null : csvRecord[i]; final Optional<DataType> dataTypeOption = schema.getDataType(rawFieldName); if (!dataTypeOption.isPresent() && dropUnknownFields) { continue; } final Object value; if (coerceTypes && dataTypeOption.isPresent()) { value = convert(rawValue, dataTypeOption.get(), rawFieldName); } else if (dataTypeOption.isPresent()) { value = convertSimpleIfPossible(rawValue, dataTypeOption.get(), rawFieldName); } else { value = rawValue; } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; } }
JacksonCSVRecordReader extends AbstractCSVRecordReader { @Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); if (recordStream.hasNext()) { String[] csvRecord = recordStream.next(); if (rawFieldNames == null) { if (!hasHeader || ignoreHeader) { rawFieldNames = schema.getFieldNames(); } else { rawFieldNames = Arrays.asList(csvRecord); if (recordStream.hasNext()) { csvRecord = recordStream.next(); } else { return null; } } } boolean foundRecord = true; if (csvRecord == null || (csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = false; while (recordStream.hasNext()) { csvRecord = recordStream.next(); if (csvRecord != null && !(csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = true; break; } } } if (!foundRecord) { return null; } final Map<String, Object> values = new HashMap<>(rawFieldNames.size() * 2); final int numFieldNames = rawFieldNames.size(); for (int i = 0; i < csvRecord.length; i++) { final String rawFieldName = numFieldNames <= i ? "unknown_field_index_" + i : rawFieldNames.get(i); String rawValue = (i >= csvRecord.length) ? null : csvRecord[i]; final Optional<DataType> dataTypeOption = schema.getDataType(rawFieldName); if (!dataTypeOption.isPresent() && dropUnknownFields) { continue; } final Object value; if (coerceTypes && dataTypeOption.isPresent()) { value = convert(rawValue, dataTypeOption.get(), rawFieldName); } else if (dataTypeOption.isPresent()) { value = convertSimpleIfPossible(rawValue, dataTypeOption.get(), rawFieldName); } else { value = rawValue; } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; } JacksonCSVRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema, final CSVFormat csvFormat, final boolean hasHeader, final boolean ignoreHeader,
                                  final String dateFormat, final String timeFormat, final String timestampFormat, final String encoding); }
JacksonCSVRecordReader extends AbstractCSVRecordReader { @Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); if (recordStream.hasNext()) { String[] csvRecord = recordStream.next(); if (rawFieldNames == null) { if (!hasHeader || ignoreHeader) { rawFieldNames = schema.getFieldNames(); } else { rawFieldNames = Arrays.asList(csvRecord); if (recordStream.hasNext()) { csvRecord = recordStream.next(); } else { return null; } } } boolean foundRecord = true; if (csvRecord == null || (csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = false; while (recordStream.hasNext()) { csvRecord = recordStream.next(); if (csvRecord != null && !(csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = true; break; } } } if (!foundRecord) { return null; } final Map<String, Object> values = new HashMap<>(rawFieldNames.size() * 2); final int numFieldNames = rawFieldNames.size(); for (int i = 0; i < csvRecord.length; i++) { final String rawFieldName = numFieldNames <= i ? "unknown_field_index_" + i : rawFieldNames.get(i); String rawValue = (i >= csvRecord.length) ? null : csvRecord[i]; final Optional<DataType> dataTypeOption = schema.getDataType(rawFieldName); if (!dataTypeOption.isPresent() && dropUnknownFields) { continue; } final Object value; if (coerceTypes && dataTypeOption.isPresent()) { value = convert(rawValue, dataTypeOption.get(), rawFieldName); } else if (dataTypeOption.isPresent()) { value = convertSimpleIfPossible(rawValue, dataTypeOption.get(), rawFieldName); } else { value = rawValue; } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; } JacksonCSVRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema, final CSVFormat csvFormat, final boolean hasHeader, final boolean ignoreHeader,
                                  final String dateFormat, final String timeFormat, final String timestampFormat, final String encoding); @Override Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields); @Override void close(); }
JacksonCSVRecordReader extends AbstractCSVRecordReader { @Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); if (recordStream.hasNext()) { String[] csvRecord = recordStream.next(); if (rawFieldNames == null) { if (!hasHeader || ignoreHeader) { rawFieldNames = schema.getFieldNames(); } else { rawFieldNames = Arrays.asList(csvRecord); if (recordStream.hasNext()) { csvRecord = recordStream.next(); } else { return null; } } } boolean foundRecord = true; if (csvRecord == null || (csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = false; while (recordStream.hasNext()) { csvRecord = recordStream.next(); if (csvRecord != null && !(csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) { foundRecord = true; break; } } } if (!foundRecord) { return null; } final Map<String, Object> values = new HashMap<>(rawFieldNames.size() * 2); final int numFieldNames = rawFieldNames.size(); for (int i = 0; i < csvRecord.length; i++) { final String rawFieldName = numFieldNames <= i ? "unknown_field_index_" + i : rawFieldNames.get(i); String rawValue = (i >= csvRecord.length) ? null : csvRecord[i]; final Optional<DataType> dataTypeOption = schema.getDataType(rawFieldName); if (!dataTypeOption.isPresent() && dropUnknownFields) { continue; } final Object value; if (coerceTypes && dataTypeOption.isPresent()) { value = convert(rawValue, dataTypeOption.get(), rawFieldName); } else if (dataTypeOption.isPresent()) { value = convertSimpleIfPossible(rawValue, dataTypeOption.get(), rawFieldName); } else { value = rawValue; } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; } JacksonCSVRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema, final CSVFormat csvFormat, final boolean hasHeader, final boolean ignoreHeader,
                                  final String dateFormat, final String timeFormat, final String timestampFormat, final String encoding); @Override Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields); @Override void close();  }