@Test public void testMultipleRecordsEscapedWithSpecialChar() throws IOException, MalformedRecordException { char delimiter = StringEscapeUtils.unescapeJava("\u0001").charAt(0); final CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withTrim().withQuote('"').withDelimiter(delimiter); final List<RecordField> fields = getDefaultFields(); fields.replaceAll(f -> f.getFieldName().equals("balance") ? new RecordField("balance", doubleDataType) : f); final RecordSchema schema = new SimpleRecordSchema(fields); try (final InputStream fis = new FileInputStream(new File("src/test/resources/csv/multi-bank-account_escapedchar.csv")); final CSVRecordReader reader = createReader(fis, schema, format)) { final Object[] firstRecord = reader.nextRecord().getValues(); final Object[] firstExpectedValues = new Object[] {"1", "John Doe", 4750.89D, "123 My Street", "My City", "MS", "11111", "USA"}; Assert.assertArrayEquals(firstExpectedValues, firstRecord); final Object[] secondRecord = reader.nextRecord().getValues(); final Object[] secondExpectedValues = new Object[] {"2", "Jane Doe", 4820.09D, "321 Your Street", "Your City", "NY", "33333", "USA"}; Assert.assertArrayEquals(secondExpectedValues, secondRecord); assertNull(reader.nextRecord()); } }
@Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); final List<RecordField> recordFields = getRecordFields(); final int numFieldNames = recordFields.size(); for (final CSVRecord csvRecord : csvParser) { final Map<String, Object> values = new LinkedHashMap<>(recordFields.size() * 2); for (int i = 0; i < csvRecord.size(); i++) { final String rawValue = csvRecord.get(i); final String rawFieldName; final DataType dataType; if (i >= numFieldNames) { if (!dropUnknownFields) { values.put("unknown_field_index_" + i, rawValue); } continue; } else { final RecordField recordField = recordFields.get(i); rawFieldName = recordField.getFieldName(); dataType = recordField.getDataType(); } final Object value; if (coerceTypes) { value = convert(rawValue, dataType, rawFieldName); } else { value = convertSimpleIfPossible(rawValue, dataType, rawFieldName); } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; }
CSVRecordReader extends AbstractCSVRecordReader { @Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); final List<RecordField> recordFields = getRecordFields(); final int numFieldNames = recordFields.size(); for (final CSVRecord csvRecord : csvParser) { final Map<String, Object> values = new LinkedHashMap<>(recordFields.size() * 2); for (int i = 0; i < csvRecord.size(); i++) { final String rawValue = csvRecord.get(i); final String rawFieldName; final DataType dataType; if (i >= numFieldNames) { if (!dropUnknownFields) { values.put("unknown_field_index_" + i, rawValue); } continue; } else { final RecordField recordField = recordFields.get(i); rawFieldName = recordField.getFieldName(); dataType = recordField.getDataType(); } final Object value; if (coerceTypes) { value = convert(rawValue, dataType, rawFieldName); } else { value = convertSimpleIfPossible(rawValue, dataType, rawFieldName); } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; } }
CSVRecordReader extends AbstractCSVRecordReader { @Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); final List<RecordField> recordFields = getRecordFields(); final int numFieldNames = recordFields.size(); for (final CSVRecord csvRecord : csvParser) { final Map<String, Object> values = new LinkedHashMap<>(recordFields.size() * 2); for (int i = 0; i < csvRecord.size(); i++) { final String rawValue = csvRecord.get(i); final String rawFieldName; final DataType dataType; if (i >= numFieldNames) { if (!dropUnknownFields) { values.put("unknown_field_index_" + i, rawValue); } continue; } else { final RecordField recordField = recordFields.get(i); rawFieldName = recordField.getFieldName(); dataType = recordField.getDataType(); } final Object value; if (coerceTypes) { value = convert(rawValue, dataType, rawFieldName); } else { value = convertSimpleIfPossible(rawValue, dataType, rawFieldName); } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; } CSVRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema, final CSVFormat csvFormat, final boolean hasHeader, final boolean ignoreHeader,
                           final String dateFormat, final String timeFormat, final String timestampFormat, final String encoding); }
CSVRecordReader extends AbstractCSVRecordReader { @Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); final List<RecordField> recordFields = getRecordFields(); final int numFieldNames = recordFields.size(); for (final CSVRecord csvRecord : csvParser) { final Map<String, Object> values = new LinkedHashMap<>(recordFields.size() * 2); for (int i = 0; i < csvRecord.size(); i++) { final String rawValue = csvRecord.get(i); final String rawFieldName; final DataType dataType; if (i >= numFieldNames) { if (!dropUnknownFields) { values.put("unknown_field_index_" + i, rawValue); } continue; } else { final RecordField recordField = recordFields.get(i); rawFieldName = recordField.getFieldName(); dataType = recordField.getDataType(); } final Object value; if (coerceTypes) { value = convert(rawValue, dataType, rawFieldName); } else { value = convertSimpleIfPossible(rawValue, dataType, rawFieldName); } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; } CSVRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema, final CSVFormat csvFormat, final boolean hasHeader, final boolean ignoreHeader,
                           final String dateFormat, final String timeFormat, final String timestampFormat, final String encoding); @Override Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields); @Override void close(); }
CSVRecordReader extends AbstractCSVRecordReader { @Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException { final RecordSchema schema = getSchema(); final List<RecordField> recordFields = getRecordFields(); final int numFieldNames = recordFields.size(); for (final CSVRecord csvRecord : csvParser) { final Map<String, Object> values = new LinkedHashMap<>(recordFields.size() * 2); for (int i = 0; i < csvRecord.size(); i++) { final String rawValue = csvRecord.get(i); final String rawFieldName; final DataType dataType; if (i >= numFieldNames) { if (!dropUnknownFields) { values.put("unknown_field_index_" + i, rawValue); } continue; } else { final RecordField recordField = recordFields.get(i); rawFieldName = recordField.getFieldName(); dataType = recordField.getDataType(); } final Object value; if (coerceTypes) { value = convert(rawValue, dataType, rawFieldName); } else { value = convertSimpleIfPossible(rawValue, dataType, rawFieldName); } values.put(rawFieldName, value); } return new MapRecord(schema, values, coerceTypes, dropUnknownFields); } return null; } CSVRecordReader(final InputStream in, final ComponentLog logger, final RecordSchema schema, final CSVFormat csvFormat, final boolean hasHeader, final boolean ignoreHeader,
                           final String dateFormat, final String timeFormat, final String timestampFormat, final String encoding); @Override Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields); @Override void close();  }