CSVSchemaParser implements SchemaParser { @Override public Schema parse(Request request) { List<Schema.SheetContent> sheetContents = new ArrayList<>(); sheetContents.add(new Schema.SheetContent(META_KEY, new ArrayList<>())); try { final DataSetMetadata metadata = request.getMetadata(); final Map<String, String> parameters = guess(request, metadata.getEncoding()); metadata.getContent().setParameters(parameters); metadata.getContent().setNbLinesInHeader(Integer.parseInt(parameters.get(HEADER_NB_LINES_PARAMETER))); List<String> header = csvFormatUtils.retrieveHeader(parameters); if (header == null || header.isEmpty()) { throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_CONTENT); } LOGGER.debug("Columns found: {}", header); LOGGER.debug("Setting default type for columns..."); int i = 0; for (String column : header) { sheetContents .stream() .filter(sheetContent -> META_KEY.equals(sheetContent.getName())) .findFirst() .get() .getColumnMetadatas() .add(column().id(i++).name(column).type(Type.STRING).build()); } } catch (Exception e) { throw new TDPException(CommonErrorCodes.UNABLE_TO_READ_CONTENT, e); } return Schema.Builder .parserResult() .sheetContents(sheetContents) .draft(false) .build(); }  @Override boolean accept(DataSetMetadata metadata); @Override Schema parse(Request request); Map<String, String> guess(SchemaParser.Request request, String encoding); static final List<Character> DEFAULT_VALID_SEPARATORS; }
@Test public void should_parse_csv() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("simple.csv")) { final String[] columns = { "first name", "last name" }; DataSetMetadata datasetMetadata = ioTestUtils.getSimpleDataSetMetadata(columns); resetParameters(datasetMetadata, ";", Arrays.asList(columns), 1, false); Schema result = csvSchemaParser.parse(new SchemaParser.Request(inputStream, datasetMetadata)); List<ColumnMetadata> actual = result.getSheetContents().get(0).getColumnMetadatas(); Assert.assertEquals(datasetMetadata.getRowMetadata().getColumns(), actual); } }
@Test public void TDP_898() throws IOException { String str = "c1;c2" + System.lineSeparator() + "1;2;false"; try (InputStream inputStream = new ByteArrayInputStream(str.getBytes())) { final String[] columns = { "c1", "c2" }; DataSetMetadata datasetMetadata = ioTestUtils.getSimpleDataSetMetadata(columns); resetParameters(datasetMetadata, ";", Arrays.asList(columns), 1, false); try { csvSchemaParser.parse(new SchemaParser.Request(inputStream, datasetMetadata)); } catch (IndexOutOfBoundsException exc) { Assert.fail("Should not throw an IndexOutOfBoundsException, when parsing!"); } } }
@Test public void shouldFindPipeSeparator() throws IOException { String str = "c1|c2|c3" + System.lineSeparator() + "1|2|false"; try (InputStream inputStream = new ByteArrayInputStream(str.getBytes())) { SchemaParser.Request request = getRequest(inputStream, "#3"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals('|', actual); } }
@Test public void shouldPassOnNbLinesHeaderParameter_TDP_5925() throws IOException { String str = "1;2;3;4;5;6\n" + "1;John;Lennon;40;10/09/1940;false\n" + "2;David;Bowie;67;01/08/1947;true\n"; try (InputStream inputStream = new ByteArrayInputStream(str.getBytes())) { SchemaParser.Request request = getRequest(inputStream, "#3"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); int actual = Integer.parseInt(parameters.get(CSVFormatFamily.HEADER_NB_LINES_PARAMETER)); assertEquals(0, actual); } }
@Test public void should_guess_best_separator() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("mixed_separators.csv")) { SchemaParser.Request request = getRequest(inputStream, "#1"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(';', actual); } }
@Test public void should_guess_best_separator_out_of_two() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("tdp-181.csv")) { SchemaParser.Request request = getRequest(inputStream, "#2"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(';', actual); } }
@Test public void should_guess_separator_with_ISO_8859_1_encoded_file() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("iso-8859-1.csv")) { SchemaParser.Request request = getRequest(inputStream, "#3"); request.getMetadata().setEncoding("ISO-8859-1"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(';', actual); } }
@Test public void should_guess_valid_separator_when_most_likely_separator_is_not_valid() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("tdp-863.csv")) { SchemaParser.Request request = getRequest(inputStream, "#4"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(';', actual); } }
@Test public void should_guess_valid_separator_from_access_log_file() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("tdp-832.csv")) { SchemaParser.Request request = getRequest(inputStream, "#5"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(' ', actual); } }
@Test public void TDP_1060() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("tdp-1060.csv")) { SchemaParser.Request request = getRequest(inputStream, "#6"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(',', actual); } }
@Test public void consistency_test() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("consistency_example.csv")) { SchemaParser.Request request = getRequest(inputStream, "#7"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(',', actual); } }
@Test public void TDP_1240_should_detect_separator_if_wrong_characters_do_not_exceed_the_threshold() throws IOException { char[] chars = new char[50]; for (int i = 0; i < 48; i++) { chars[i] = 0; } chars[48] = ';'; chars[49] = '\n'; try (InputStream inputStream = IOUtils.toInputStream(new String(chars))) { SchemaParser.Request request = getRequest(inputStream, "#8"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(';', actual); } chars = new char[50]; for (int i = 0; i < 48; i++) { chars[i] = 65533; } chars[48] = ';'; chars[49] = '\n'; try (InputStream inputStream = IOUtils.toInputStream(new String(chars))) { SchemaParser.Request request = getRequest(inputStream, "#9"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(';', actual); } }
@Test public void TDP_1240_should_detect_separator_if_wrong_characters_exceed_threshold() throws IOException { char[] chars = new char[51]; for (int i = 0; i < 49; i++) { chars[i] = 0; } chars[49] = ';'; chars[50] = '\n'; try (InputStream inputStream = IOUtils.toInputStream(new String(chars))) { SchemaParser.Request request = getRequest(inputStream, "#10"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(';', actual); } chars = new char[51]; for (int i = 0; i < 49; i++) { chars[i] = 65533; } chars[49] = ';'; chars[50] = '\n'; try (InputStream inputStream = IOUtils.toInputStream(new String(chars))) { SchemaParser.Request request = getRequest(inputStream, "#11"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(';', actual); } }
@Test public void should_detect_comma_separator_when_no_separator_detected() throws IOException { char[] chars = new char[5]; for (int i = 0; i < 5; i++) { chars[i] = 'A'; } try (InputStream inputStream = IOUtils.toInputStream(new String(chars))) { SchemaParser.Request request = getRequest(inputStream, "#12"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); assertEquals(',', actual); } }
@Test public void TDP_1259() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("tdp-1259.csv")) { SchemaParser.Request request = getRequest(inputStream, "#1"); request.getMetadata().setEncoding("UTF-8"); csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); List<String> header = csvFormatUtils.retrieveHeader(parameters); assertEquals(';', actual); List<String> expected = Arrays.asList("id", "first_name", "last_name", "email", "job_title", "company", "city", "state", "country", "date", "campaign_id", "lead_score", "registration", "city", "birth", "nbCommands", "id", "first_name", "last_name", "email", "job_title", "company", "city", "state", "country", "date", "campaign_id", "lead_score", "registration", "city", "birth", "nbCommands"); assertEquals(expected, header); } }
@Test public void TDP_2171() throws IOException { try (InputStream inputStream = this.getClass().getResourceAsStream("TDP-2171.csv")) { SchemaParser.Request request = getRequest(inputStream, "#1"); request.getMetadata().setEncoding("UTF-8"); Schema schema = csvSchemaParser.parse(request); final Map<String, String> parameters = request.getMetadata().getContent().getParameters(); char actual = parameters.get(CSVFormatFamily.SEPARATOR_PARAMETER).charAt(0); List<String> header = csvFormatUtils.retrieveHeader(parameters); assertEquals(',', actual); assertEquals(92, header.size()); } }