DataTypeQualityAnalyzer extends QualityAnalyzer<ValueQualityStatistics, DataTypeEnum[]> { @Override public List<ValueQualityStatistics> getResult() { return results; } DataTypeQualityAnalyzer(DataTypeEnum[] types, boolean isStoreInvalidValues);  DataTypeQualityAnalyzer(DataTypeEnum... types); @Override void init(); @Override boolean analyze(String... record); @Override void end(); @Override List<ValueQualityStatistics> getResult(); @Override void close();  }
@Test public void testTwoColumnsInvalid() { DataTypeQualityAnalyzer qualityAnalyzer = new DataTypeQualityAnalyzer(DataTypeEnum.INTEGER, DataTypeEnum.DOUBLE); populateAnalyzeData(qualityAnalyzer); ValueQualityStatistics valueQuality = qualityAnalyzer.getResult().get(0); assertEquals(2, valueQuality.getInvalidCount()); assertEquals(3, valueQuality.getValidCount()); Set<String> invalidValues = valueQuality.getInvalidValues(); assertTrue(invalidValues.contains("str")); assertTrue(invalidValues.contains("another str")); valueQuality = qualityAnalyzer.getResult().get(1); assertEquals(1, valueQuality.getInvalidCount()); assertEquals(2, valueQuality.getValidCount()); assertEquals(2, valueQuality.getEmptyCount()); invalidValues = valueQuality.getInvalidValues(); assertEquals(1, invalidValues.size()); assertTrue(invalidValues.contains("a")); }
@Test public void testTwoColumnsValidAndEmpty() throws Exception { DataTypeQualityAnalyzer qualityAnalyzer = new DataTypeQualityAnalyzer(DataTypeEnum.STRING, DataTypeEnum.STRING); populateAnalyzeData(qualityAnalyzer); ValueQualityStatistics valueQuality = qualityAnalyzer.getResult().get(0); assertEquals(0, valueQuality.getInvalidCount()); assertEquals(5, valueQuality.getValidCount()); Set<String> invalidValues = valueQuality.getInvalidValues(); assertEquals(0, invalidValues.size()); valueQuality = qualityAnalyzer.getResult().get(1); assertEquals(0, valueQuality.getInvalidCount()); assertEquals(3, valueQuality.getValidCount()); assertEquals(2, valueQuality.getEmptyCount()); invalidValues = valueQuality.getInvalidValues(); assertEquals(0, invalidValues.size()); }
@Test public void testNoneStrings_double() { DataTypeQualityAnalyzer qualityAnalyzer = new DataTypeQualityAnalyzer(DataTypeEnum.DOUBLE); populateAnalyzerNoneString(qualityAnalyzer); ValueQualityStatistics valueQuality = qualityAnalyzer.getResult().get(0); assertEquals(1, valueQuality.getInvalidCount()); assertEquals(5, valueQuality.getValidCount()); Set<String> invalidValues = valueQuality.getInvalidValues(); assertEquals(1, invalidValues.size()); assertTrue(invalidValues.contains("str")); }
@Test public void testNoneStrings_string() throws Exception { DataTypeQualityAnalyzer qualityAnalyzer = new DataTypeQualityAnalyzer(DataTypeEnum.STRING); populateAnalyzerNoneString(qualityAnalyzer); ValueQualityStatistics valueQuality = qualityAnalyzer.getResult().get(0); assertEquals(0, valueQuality.getInvalidCount()); assertEquals(6, valueQuality.getValidCount()); Set<String> invalidValues = valueQuality.getInvalidValues(); assertEquals(0, invalidValues.size()); }
@Test public void testNoneStrings_integer() { DataTypeQualityAnalyzer qualityAnalyzer = new DataTypeQualityAnalyzer(DataTypeEnum.INTEGER); populateAnalyzerNoneString(qualityAnalyzer); ValueQualityStatistics valueQuality = qualityAnalyzer.getResult().get(0); assertEquals(4, valueQuality.getInvalidCount()); assertEquals(2, valueQuality.getValidCount()); Set<String> invalidValues = valueQuality.getInvalidValues(); assertEquals(4, invalidValues.size()); assertTrue(invalidValues.contains("1.0")); assertTrue(invalidValues.contains("0.02")); assertTrue(invalidValues.contains("2.88888888888888888888888")); assertTrue(invalidValues.contains("str")); }
@Test public void testNumbers_double() { DataTypeQualityAnalyzer qualityAnalyzer = new DataTypeQualityAnalyzer(DataTypeEnum.DOUBLE); populateAnalyzerWithNumbers(qualityAnalyzer); ValueQualityStatistics valueQuality = qualityAnalyzer.getResult().get(0); assertEquals(0, valueQuality.getInvalidCount()); assertEquals(5, valueQuality.getValidCount()); Set<String> invalidValues = valueQuality.getInvalidValues(); assertEquals(0, invalidValues.size()); }
@Test public void testNumbers_string() { DataTypeQualityAnalyzer qualityAnalyzer = new DataTypeQualityAnalyzer(DataTypeEnum.STRING); populateAnalyzerWithNumbers(qualityAnalyzer); ValueQualityStatistics valueQuality = qualityAnalyzer.getResult().get(0); assertEquals(0, valueQuality.getInvalidCount()); assertEquals(5, valueQuality.getValidCount()); Set<String> invalidValues = valueQuality.getInvalidValues(); assertEquals(0, invalidValues.size()); }