@Test public void testEncodingDetection() { String id = UUID.randomUUID().toString(); final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build(); dataSetMetadataRepository.save(metadata); contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../wave_lab_utf16_LE.txt")); formatAnalysis.analyze(id); final DataSetMetadata actual = dataSetMetadataRepository.get(id); assertThat(actual, notNullValue()); assertThat(actual.getContent().getFormatFamilyId(), is(CSVFormatFamily.BEAN_ID)); assertThat(actual.getContent().getMediaType(), is("text/csv")); assertThat(actual.getContent().getParameters().get("SEPARATOR"), is("\t")); assertThat(actual.getEncoding(), is("UTF-16LE")); }
@Override public void analyze(String dataSetId) { if (StringUtils.isEmpty(dataSetId)) { throw new IllegalArgumentException("Data set id cannot be null or empty."); } final Marker marker = Markers.dataset(dataSetId); DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId); datasetLock.lock(); try { DataSetMetadata metadata = repository.get(dataSetId); if (metadata != null) { Format detectedFormat = null; for (byte[] bom : BOMS) { try (InputStream content = store.getAsRaw(metadata, 10)) { detectedFormat = detector.detect(addBOM(content, bom)); } catch (IOException e) { throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_CONTENT, e); } if (detectedFormat != null && !(detectedFormat.getFormatFamily() instanceof UnsupportedFormatFamily)) { break; } } LOG.debug(marker, "using {} to parse the dataset", detectedFormat); verifyFormat(detectedFormat); internalUpdateMetadata(metadata, detectedFormat); LOG.debug(marker, "format analysed for dataset"); } else { LOG.info(marker, "Data set no longer exists."); } } finally { datasetLock.unlock(); } }
FormatAnalysis implements SynchronousDataSetAnalyzer { @Override public void analyze(String dataSetId) { if (StringUtils.isEmpty(dataSetId)) { throw new IllegalArgumentException("Data set id cannot be null or empty."); } final Marker marker = Markers.dataset(dataSetId); DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId); datasetLock.lock(); try { DataSetMetadata metadata = repository.get(dataSetId); if (metadata != null) { Format detectedFormat = null; for (byte[] bom : BOMS) { try (InputStream content = store.getAsRaw(metadata, 10)) { detectedFormat = detector.detect(addBOM(content, bom)); } catch (IOException e) { throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_CONTENT, e); } if (detectedFormat != null && !(detectedFormat.getFormatFamily() instanceof UnsupportedFormatFamily)) { break; } } LOG.debug(marker, "using {} to parse the dataset", detectedFormat); verifyFormat(detectedFormat); internalUpdateMetadata(metadata, detectedFormat); LOG.debug(marker, "format analysed for dataset"); } else { LOG.info(marker, "Data set no longer exists."); } } finally { datasetLock.unlock(); } } }
FormatAnalysis implements SynchronousDataSetAnalyzer { @Override public void analyze(String dataSetId) { if (StringUtils.isEmpty(dataSetId)) { throw new IllegalArgumentException("Data set id cannot be null or empty."); } final Marker marker = Markers.dataset(dataSetId); DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId); datasetLock.lock(); try { DataSetMetadata metadata = repository.get(dataSetId); if (metadata != null) { Format detectedFormat = null; for (byte[] bom : BOMS) { try (InputStream content = store.getAsRaw(metadata, 10)) { detectedFormat = detector.detect(addBOM(content, bom)); } catch (IOException e) { throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_CONTENT, e); } if (detectedFormat != null && !(detectedFormat.getFormatFamily() instanceof UnsupportedFormatFamily)) { break; } } LOG.debug(marker, "using {} to parse the dataset", detectedFormat); verifyFormat(detectedFormat); internalUpdateMetadata(metadata, detectedFormat); LOG.debug(marker, "format analysed for dataset"); } else { LOG.info(marker, "Data set no longer exists."); } } finally { datasetLock.unlock(); } }  }
FormatAnalysis implements SynchronousDataSetAnalyzer { @Override public void analyze(String dataSetId) { if (StringUtils.isEmpty(dataSetId)) { throw new IllegalArgumentException("Data set id cannot be null or empty."); } final Marker marker = Markers.dataset(dataSetId); DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId); datasetLock.lock(); try { DataSetMetadata metadata = repository.get(dataSetId); if (metadata != null) { Format detectedFormat = null; for (byte[] bom : BOMS) { try (InputStream content = store.getAsRaw(metadata, 10)) { detectedFormat = detector.detect(addBOM(content, bom)); } catch (IOException e) { throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_CONTENT, e); } if (detectedFormat != null && !(detectedFormat.getFormatFamily() instanceof UnsupportedFormatFamily)) { break; } } LOG.debug(marker, "using {} to parse the dataset", detectedFormat); verifyFormat(detectedFormat); internalUpdateMetadata(metadata, detectedFormat); LOG.debug(marker, "format analysed for dataset"); } else { LOG.info(marker, "Data set no longer exists."); } } finally { datasetLock.unlock(); } }  @Override void analyze(String dataSetId); void update(DataSetMetadata original, DataSetMetadata updated); @Override int order(); }
FormatAnalysis implements SynchronousDataSetAnalyzer { @Override public void analyze(String dataSetId) { if (StringUtils.isEmpty(dataSetId)) { throw new IllegalArgumentException("Data set id cannot be null or empty."); } final Marker marker = Markers.dataset(dataSetId); DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId); datasetLock.lock(); try { DataSetMetadata metadata = repository.get(dataSetId); if (metadata != null) { Format detectedFormat = null; for (byte[] bom : BOMS) { try (InputStream content = store.getAsRaw(metadata, 10)) { detectedFormat = detector.detect(addBOM(content, bom)); } catch (IOException e) { throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_CONTENT, e); } if (detectedFormat != null && !(detectedFormat.getFormatFamily() instanceof UnsupportedFormatFamily)) { break; } } LOG.debug(marker, "using {} to parse the dataset", detectedFormat); verifyFormat(detectedFormat); internalUpdateMetadata(metadata, detectedFormat); LOG.debug(marker, "format analysed for dataset"); } else { LOG.info(marker, "Data set no longer exists."); } } finally { datasetLock.unlock(); } }  @Override void analyze(String dataSetId); void update(DataSetMetadata original, DataSetMetadata updated); @Override int order();  }