AvroDataTypeQualityAnalyzer extends AvroQualityAnalyzer { @Override public void init() { frequentDatePatterns.clear(); qualityResults.clear(); knownDataTypeCaches.clear(); } AvroDataTypeQualityAnalyzer(boolean isStoreInvalidValues);  AvroDataTypeQualityAnalyzer(); @Override void init(); @Override void init(Schema semanticSchema); @Override boolean analyze(IndexedRecord record); @Override Stream<IndexedRecord> analyze(Stream<IndexedRecord> records);  }
@Test public void testSemanticSchemaNotMatchingRecord() { try { String path = AvroDataTypeDiscoveryAnalyzerTest.class.getResource("../sample/date.avro").getPath(); File dateAvroFile = new File(path); DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(dateAvroFile, new GenericDatumReader<>()); AvroDataTypeDiscoveryAnalyzer discoveryAnalyzer = new AvroDataTypeDiscoveryAnalyzer(); discoveryAnalyzer.init(dateAvroReader.getSchema()); dateAvroReader.forEach(discoveryAnalyzer::analyze); Schema discoveryResult = discoveryAnalyzer.getResult(); AvroDataTypeQualityAnalyzer qualityAnalyzer = new AvroDataTypeQualityAnalyzer(); qualityAnalyzer.init(discoveryResult); dateAvroReader.sync(0); dateAvroReader.forEach(qualityAnalyzer::analyze); Schema validationResult = qualityAnalyzer.getResult(); assertNotNull(validationResult); } catch (IOException e) { e.printStackTrace(); } }
@Test public void testAvroDataTypeQualityAnalyzerOnPrimitiveSchemas() { try { String path = AvroDataTypeDiscoveryAnalyzerTest.class.getResource("../sample/primitive").getPath(); File primitiveFolder = new File(path); for (final File fileEntry : Objects.requireNonNull(primitiveFolder.listFiles())) { DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); AvroDataTypeDiscoveryAnalyzer discoveryAnalyzer = new AvroDataTypeDiscoveryAnalyzer(); discoveryAnalyzer.init(dateAvroReader.getSchema()); dateAvroReader.forEach(discoveryAnalyzer::analyze); Schema discoveryResult = discoveryAnalyzer.getResult(); AvroDataTypeQualityAnalyzer qualityAnalyzer = new AvroDataTypeQualityAnalyzer(); qualityAnalyzer.init(discoveryResult); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); dateAvroReader.forEach(qualityAnalyzer::analyze); Schema validationResult = qualityAnalyzer.getResult(); assertNotNull(validationResult); } } catch (IOException e) { e.printStackTrace(); } }
@Test public void testAvroDataTypeQualityAnalyzerOnStructureSchemas() { try { String path = AvroDataTypeDiscoveryAnalyzerTest.class.getResource("../sample/structure").getPath(); File primitiveFolder = new File(path); for (final File fileEntry : Objects.requireNonNull(primitiveFolder.listFiles())) { DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); AvroDataTypeDiscoveryAnalyzer discoveryAnalyzer = new AvroDataTypeDiscoveryAnalyzer(); discoveryAnalyzer.init(dateAvroReader.getSchema()); dateAvroReader.forEach(discoveryAnalyzer::analyze); Schema discoveryResult = discoveryAnalyzer.getResult(); AvroDataTypeQualityAnalyzer qualityAnalyzer = new AvroDataTypeQualityAnalyzer(); qualityAnalyzer.init(discoveryResult); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); dateAvroReader.forEach(qualityAnalyzer::analyze); Schema validationResult = qualityAnalyzer.getResult(); assertNotNull(validationResult); } } catch (IOException e) { e.printStackTrace(); } }
@Test public void testAvroDataTypeQualityAnalyzerOnComplexSchemas() { try { String path = AvroDataTypeDiscoveryAnalyzerTest.class.getResource("../sample/complex").getPath(); File primitiveFolder = new File(path); for (final File fileEntry : Objects.requireNonNull(primitiveFolder.listFiles())) { DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); AvroDataTypeDiscoveryAnalyzer discoveryAnalyzer = new AvroDataTypeDiscoveryAnalyzer(); discoveryAnalyzer.init(dateAvroReader.getSchema()); dateAvroReader.forEach(discoveryAnalyzer::analyze); Schema discoveryResult = discoveryAnalyzer.getResult(); AvroDataTypeQualityAnalyzer qualityAnalyzer = new AvroDataTypeQualityAnalyzer(); qualityAnalyzer.init(discoveryResult); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); dateAvroReader.forEach(qualityAnalyzer::analyze); Schema validationResult = qualityAnalyzer.getResult(); assertNotNull(validationResult); } } catch (IOException e) { e.printStackTrace(); } }
@Test @Ignore public void testAvroDataTypeQualityAnalyzerOnBigSamples() { try { String path = AvroDataTypeDiscoveryAnalyzerTest.class.getResource("../sample").getPath(); File primitiveFolder = new File(path); for (final File fileEntry : Objects.requireNonNull(primitiveFolder.listFiles())) { if (fileEntry.isFile()) { System.out.println(fileEntry); DataFileReader<GenericRecord> dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); AvroDataTypeDiscoveryAnalyzer discoveryAnalyzer = new AvroDataTypeDiscoveryAnalyzer(); discoveryAnalyzer.init(dateAvroReader.getSchema()); dateAvroReader.forEach(discoveryAnalyzer::analyze); Schema discoveryResult = discoveryAnalyzer.getResult(); System.out.println(discoveryResult.toString()); AvroDataTypeQualityAnalyzer qualityAnalyzer = new AvroDataTypeQualityAnalyzer(); qualityAnalyzer.init(discoveryResult); dateAvroReader = new DataFileReader<>(fileEntry, new GenericDatumReader<>()); dateAvroReader.forEach(qualityAnalyzer::analyze); Schema validationResult = qualityAnalyzer.getResult(); System.out.println(validationResult.toString()); assertNotNull(validationResult); } } } catch (IOException e) { e.printStackTrace(); } }