CSVSerializer implements Serializer { @Override public InputStream serialize(InputStream rawContent, DataSetMetadata metadata, long limit) { try { PipedInputStream pipe = new PipedInputStream(); PipedOutputStream jsonOutput = new PipedOutputStream(pipe); Runnable r = () -> { final Map<String, String> parameters = metadata.getContent().getParameters(); final String separator = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER); final char actualSeparator = separator.charAt(0); final char textEnclosureChar = getFromParameters(parameters, TEXT_ENCLOSURE_CHAR, defaultTextEnclosure); final char escapeChar = getFromParameters(parameters, CSVFormatFamily.ESCAPE_CHAR, defaultEscapeChar); try (InputStreamReader input = new InputStreamReader(rawContent, metadata.getEncoding()); CSVReader reader = new CSVReader(input, actualSeparator, textEnclosureChar, escapeChar)) { JsonGenerator generator = new JsonFactory().createGenerator(jsonOutput); int i = 0; while (i++ < metadata.getContent().getNbLinesInHeader()) { reader.readNext(); } generator.writeStartArray(); writeLineContent(reader, metadata, generator, separator, limit); generator.writeEndArray(); generator.flush(); } catch (Exception e) { LOGGER.debug("Unable to continue serialization for {}. Skipping remaining content.", metadata.getId(), e); } finally { try { jsonOutput.close(); } catch (IOException e) { LOGGER.error("Unable to close output", e); } } }; executor.execute(r); return pipe; } catch (IOException e) { throw new TDPException(CommonErrorCodes.UNABLE_TO_SERIALIZE_TO_JSON, e); } }  @Override InputStream serialize(InputStream rawContent, DataSetMetadata metadata, long limit);  }
@Test public void should_serialize_standard_csv() throws IOException { InputStream inputStream = this.getClass().getResourceAsStream("simple.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("first name", "last name"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); InputStream expected = this.getClass().getResourceAsStream("simple.csv_expected.json"); Assert.assertThat(actual, sameJSONAsFile(expected)); }
@Test public void should_serialize_csv_with_missing_values() throws IOException { InputStream inputStream = this.getClass().getResourceAsStream("missing_values.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("character", "actor", "active"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); InputStream expected = this.getClass().getResourceAsStream("missing_values.csv_expected.json"); Assert.assertThat(actual, sameJSONAsFile(expected)); }
@Test public void should_serialize_csv_with_additional_values() throws IOException { InputStream inputStream = this.getClass().getResourceAsStream("additional_values.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("name", "email"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); InputStream expected = this.getClass().getResourceAsStream("additional_values.csv_expected.json"); Assert.assertThat(actual, sameJSONAsFile(expected)); }
@Test public void should_serialize_csv_with_two_lines_header() throws IOException { InputStream inputStream = this.getClass().getResourceAsStream("two_lines_header.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("first name", "last name"); datasetMetadata.getContent().setNbLinesInHeader(2); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); InputStream expected = this.getClass().getResourceAsStream("simple.csv_expected.json"); Assert.assertThat(actual, sameJSONAsFile(expected)); }
@Test public void should_serialize_csv_with_the_specified_encoding() throws IOException { InputStream inputStream = this.getClass().getResourceAsStream("x_mac_roman.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("Titre", "Pr�nom"); datasetMetadata.setEncoding("x-MacRoman"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); String expected = "[{\"0000\":\",\\\"GÈrard\",\"0001\":null}]"; Assert.assertEquals(expected, actual); }
@Test public void should_serialize_csv_with_backslash() throws IOException { InputStream inputStream = this.getClass().getResourceAsStream("tdp-1623_backslash_not_imported.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("City"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); String expected = "[{\"0000\":\"Carson City\\\\Seine\"}]"; Assert.assertEquals(expected, actual); }
@Test public void shouldManageSpecificTextEnclosureChar() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("with_quote_in_text.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("City", "code", "Description"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(TEXT_ENCLOSURE_CHAR, "¤"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString(this.getClass().getResourceAsStream("with_quote_in_text.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void shouldManageSpecificEscapeChar() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("with_another_escape_char.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("City", "code", "Description"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(ESCAPE_CHAR, "|"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString(this.getClass().getResourceAsStream("with_another_escape_char.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void should_use_custom_import_parameters_double_quote_escape() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("column1", "column2"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(ESCAPE_CHAR, "\""); datasetMetadata.getContent().addParameter(TEXT_ENCLOSURE_CHAR, "+"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString( this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import_double_quote_escape.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void should_use_custom_import_parameters_backslash_escape() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("column1", "column2"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(ESCAPE_CHAR, "\\"); datasetMetadata.getContent().addParameter(TEXT_ENCLOSURE_CHAR, "+"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString( this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import_backslash_escape.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void should_use_custom_import_parameters_default_empty_escape() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("column1", "column2"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(TEXT_ENCLOSURE_CHAR, "+"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString( this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import_default_escape.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void should_use_custom_import_parameters_empty_escape() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("column1", "column2"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(ESCAPE_CHAR, ""); datasetMetadata.getContent().addParameter(TEXT_ENCLOSURE_CHAR, "+"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString( this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import_default_escape.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void should_use_custom_import_parameters() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("column1", "column2"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(ESCAPE_CHAR, "'"); datasetMetadata.getContent().addParameter(TEXT_ENCLOSURE_CHAR, "+"); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString(this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void should_use_custom_import_parameters_empty_enclosure() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("column1", "column2"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(ESCAPE_CHAR, "\""); datasetMetadata.getContent().addParameter(TEXT_ENCLOSURE_CHAR, ""); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString( this.getClass().getResourceAsStream("tdp-4602_custom_param_csv_import_empty_enclosure.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void should_use_custom_import_parameters_double_quote() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("test_4579_doublequote_import.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("City", "code", "Description"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); datasetMetadata.getContent().addParameter(ESCAPE_CHAR, ""); datasetMetadata.getContent().addParameter(TEXT_ENCLOSURE_CHAR, ""); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString(this.getClass().getResourceAsStream("test_4579_doublequote_import.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }
@Test public void should_use_custom_import_parameters_pb_double_quote() throws Exception { InputStream inputStream = this.getClass().getResourceAsStream("test_4579_doublequote_import.csv"); DataSetMetadata datasetMetadata = getSimpleDataSetMetadata("City", "code", "Description"); datasetMetadata.getContent().addParameter(SEPARATOR_PARAMETER, ","); InputStream input = serializer.serialize(inputStream, datasetMetadata, -1); String actual = IOUtils.toString(input, UTF_8); final String expected = IOUtils.toString(this.getClass().getResourceAsStream("test_4579_doublequote_import_pb.json"), UTF_8); JSONAssert.assertEquals(expected, actual, false); }