FormatAnalysis implements SynchronousDataSetAnalyzer { @Override public void analyze(String dataSetId) { if (StringUtils.isEmpty(dataSetId)) { throw new IllegalArgumentException("Data set id cannot be null or empty."); } final Marker marker = Markers.dataset(dataSetId); DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId); datasetLock.lock(); try { DataSetMetadata metadata = repository.get(dataSetId); if (metadata != null) { Format detectedFormat = null; for (byte[] bom : BOMS) { try (InputStream content = store.getAsRaw(metadata, 10)) { detectedFormat = detector.detect(addBOM(content, bom)); } catch (IOException e) { throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_CONTENT, e); } if (detectedFormat != null && !(detectedFormat.getFormatFamily() instanceof UnsupportedFormatFamily)) { break; } } LOG.debug(marker, "using {} to parse the dataset", detectedFormat); verifyFormat(detectedFormat); internalUpdateMetadata(metadata, detectedFormat); LOG.debug(marker, "format analysed for dataset"); } else { LOG.info(marker, "Data set no longer exists."); } } finally { datasetLock.unlock(); } }  @Override void analyze(String dataSetId); void update(DataSetMetadata original, DataSetMetadata updated); @Override int order();  }
@Test(expected = IllegalArgumentException.class) public void testNullArgument() { formatAnalysis.analyze(null); }
@Test public void testNoDataSetFound() { String dataSetId = UUID.randomUUID().toString(); formatAnalysis.analyze(dataSetId); assertThat(dataSetMetadataRepository.get(dataSetId), nullValue()); }
@Test public void testCSVAnalysis() { String id = UUID.randomUUID().toString(); final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build(); dataSetMetadataRepository.save(metadata); contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../avengers.csv")); formatAnalysis.analyze(id); final DataSetMetadata actual = dataSetMetadataRepository.get(id); assertThat(actual, notNullValue()); assertThat(actual.getContent().getFormatFamilyId(), is(CSVFormatFamily.BEAN_ID)); assertThat(actual.getContent().getMediaType(), is("text/csv")); assertThat(actual.getContent().getParameters().get("SEPARATOR"), is(";")); }
@Test public void testEncodingDetection() { String id = UUID.randomUUID().toString(); final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build(); dataSetMetadataRepository.save(metadata); contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../wave_lab_utf16_LE.txt")); formatAnalysis.analyze(id); final DataSetMetadata actual = dataSetMetadataRepository.get(id); assertThat(actual, notNullValue()); assertThat(actual.getContent().getFormatFamilyId(), is(CSVFormatFamily.BEAN_ID)); assertThat(actual.getContent().getMediaType(), is("text/csv")); assertThat(actual.getContent().getParameters().get("SEPARATOR"), is("\t")); assertThat(actual.getEncoding(), is("UTF-16LE")); }
@Test public void testEncodingDetection_UTF16LE_WithoutBOM() { String id = UUID.randomUUID().toString(); final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build(); dataSetMetadataRepository.save(metadata); contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../utf16_LE_without_bom.txt")); formatAnalysis.analyze(id); final DataSetMetadata actual = dataSetMetadataRepository.get(id); assertThat(actual, notNullValue()); assertThat(actual.getContent().getFormatFamilyId(), is(CSVFormatFamily.BEAN_ID)); assertThat(actual.getContent().getMediaType(), is("text/csv")); assertThat(actual.getContent().getParameters().get("SEPARATOR"), is(",")); assertThat(actual.getEncoding(), is("UTF-16LE")); }
@Test public void test_TDP_690() { String id = UUID.randomUUID().toString(); final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build(); dataSetMetadataRepository.save(metadata); contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../wave_lab_utf16_LE.txt")); formatAnalysis.analyze(id); final DataSetMetadata actual = dataSetMetadataRepository.get(id); Stream<DataSetRow> content = contentStore.stream(actual); final long emptyRows = content.filter(DataSetRow::isEmpty).count(); assertThat(emptyRows, is(0L)); }
@Test public void testXLSXAnalysis() { String id = UUID.randomUUID().toString(); final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build(); dataSetMetadataRepository.save(metadata); contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../tagada.xls")); formatAnalysis.analyze(id); final DataSetMetadata actual = dataSetMetadataRepository.get(id); assertThat(actual, notNullValue()); assertThat(actual.getContent().getFormatFamilyId(), is(XlsFormatFamily.BEAN_ID)); assertThat(actual.getContent().getMediaType(), is("application/vnd.ms-excel")); assertThat(actual.getContent().getParameters().isEmpty(), is(true)); }