AvroDataTypeQualityAnalyzer extends AvroQualityAnalyzer { @Override public boolean analyze(IndexedRecord record) { analyzeRecord(record); return true; } AvroDataTypeQualityAnalyzer(boolean isStoreInvalidValues);  AvroDataTypeQualityAnalyzer(); @Override void init(); @Override void init(Schema semanticSchema); @Override boolean analyze(IndexedRecord record); @Override Stream<IndexedRecord> analyze(Stream<IndexedRecord> records);  }
@Test public void testNull() { analyzer.analyze((IndexedRecord) null); Schema result = analyzer.getResult(); assertNotNull(result); Map<String, Long> prop = (Map) result.getObjectProp(GLOBAL_QUALITY_PROP_NAME); checkQuality(prop, 0, 0, 0, 0); }
@Test public void testGlobalQuality() { GenericRecord[] records = loadPersons("alice", "bob", "charlie"); for (GenericRecord record : records) { analyzer.analyze(record); } Schema result = analyzer.getResult(); assertNotNull(result); Map<String, Long> prop = (Map) result.getObjectProp(GLOBAL_QUALITY_PROP_NAME); checkQuality(prop, 15, 3, 2, 20); }
@Test public void testSimpleFields() throws IOException, URISyntaxException { GenericRecord[] records = loadPersons("alice", "bob", "charlie"); Iterator<IndexedRecord> outRecords = analyzer.analyze(Arrays.stream(records)).iterator(); int count = 0; while (outRecords.hasNext()) { GenericRecord out = (GenericRecord) outRecords.next(); GenericData.Record firstnameRecord = (GenericData.Record) out.get("firstname"); int validity = (int) firstnameRecord.get("validity"); assertEquals(1, validity); count++; } assertEquals(records.length, count); Schema result = analyzer.getResult(); assertNotNull(result); Map<String, Long> prop = (Map) result.getField("firstname").schema().getObjectProp(QUALITY_PROP_NAME); checkQuality(prop, 3, 0, 0, 3); }
@Test public void testUnion() throws IOException, URISyntaxException { GenericRecord[] records = loadPersons("alice", "bob", "charlie"); List<IndexedRecord> outRecords = analyzer.analyze(Arrays.stream(records)).collect(Collectors.toList()); Schema result = analyzer.getResult(); assertNotNull(result); Schema zipcodeSchema = result.getField("location").schema().getField("zipcode").schema(); Schema specificZipcodeSchema = zipcodeSchema.getTypes().stream().filter(s -> s.getType() == Schema.Type.STRING).findFirst().get(); Map<String, Long> prop = (Map) specificZipcodeSchema.getObjectProp(QUALITY_PROP_NAME); checkQuality(prop, 1, 0, 0, 1); specificZipcodeSchema = zipcodeSchema.getTypes().stream().filter(s -> s.getType() == Schema.Type.NULL).findFirst().get(); prop = (Map) specificZipcodeSchema.getObjectProp(QUALITY_PROP_NAME); checkQuality(prop, 0, 0, 1, 1); specificZipcodeSchema = zipcodeSchema.getTypes().stream().filter(s -> s.getType() == Schema.Type.RECORD).findFirst().get(); Schema codeSchema = specificZipcodeSchema.getField("code").schema(); prop = (Map) codeSchema.getObjectProp(QUALITY_PROP_NAME); checkQuality(prop, 1, 0, 0, 1); }