@Test public void testAddKey(@TempDir java.nio.file.Path tempDir) throws IOException { List<String> rowKeys = new ArrayList<>(); for (int i = 0; i < 1000; i++) { rowKeys.add(UUID.randomUUID().toString()); } String filePath = tempDir.resolve("test.parquet").toAbsolutePath().toString(); Schema schema = HoodieAvroUtils.getRecordKeySchema(); BloomFilter filter = BloomFilterFactory.createBloomFilter( 1000, 0.0001, 10000, BloomFilterTypeCode.SIMPLE.name()); HoodieAvroWriteSupport writeSupport = new HoodieAvroWriteSupport( new AvroSchemaConverter().convert(schema), schema, filter); ParquetWriter writer = new ParquetWriter(new Path(filePath), writeSupport, CompressionCodecName.GZIP, 120 * 1024 * 1024, ParquetWriter.DEFAULT_PAGE_SIZE); for (String rowKey : rowKeys) { GenericRecord rec = new GenericData.Record(schema); rec.put(HoodieRecord.RECORD_KEY_METADATA_FIELD, rowKey); writer.write(rec); writeSupport.add(rowKey); } writer.close(); }
public void add(String recordKey) { this.bloomFilter.add(recordKey); if (minRecordKey != null) { minRecordKey = minRecordKey.compareTo(recordKey) <= 0 ? minRecordKey : recordKey; } else { minRecordKey = recordKey; } if (maxRecordKey != null) { maxRecordKey = maxRecordKey.compareTo(recordKey) >= 0 ? maxRecordKey : recordKey; } else { maxRecordKey = recordKey; } }
HoodieAvroWriteSupport extends AvroWriteSupport { public void add(String recordKey) { this.bloomFilter.add(recordKey); if (minRecordKey != null) { minRecordKey = minRecordKey.compareTo(recordKey) <= 0 ? minRecordKey : recordKey; } else { minRecordKey = recordKey; } if (maxRecordKey != null) { maxRecordKey = maxRecordKey.compareTo(recordKey) >= 0 ? maxRecordKey : recordKey; } else { maxRecordKey = recordKey; } } }
HoodieAvroWriteSupport extends AvroWriteSupport { public void add(String recordKey) { this.bloomFilter.add(recordKey); if (minRecordKey != null) { minRecordKey = minRecordKey.compareTo(recordKey) <= 0 ? minRecordKey : recordKey; } else { minRecordKey = recordKey; } if (maxRecordKey != null) { maxRecordKey = maxRecordKey.compareTo(recordKey) >= 0 ? maxRecordKey : recordKey; } else { maxRecordKey = recordKey; } } HoodieAvroWriteSupport(MessageType schema, Schema avroSchema, BloomFilter bloomFilter); }
HoodieAvroWriteSupport extends AvroWriteSupport { public void add(String recordKey) { this.bloomFilter.add(recordKey); if (minRecordKey != null) { minRecordKey = minRecordKey.compareTo(recordKey) <= 0 ? minRecordKey : recordKey; } else { minRecordKey = recordKey; } if (maxRecordKey != null) { maxRecordKey = maxRecordKey.compareTo(recordKey) >= 0 ? maxRecordKey : recordKey; } else { maxRecordKey = recordKey; } } HoodieAvroWriteSupport(MessageType schema, Schema avroSchema, BloomFilter bloomFilter); @Override WriteSupport.FinalizedWriteContext finalizeWrite(); void add(String recordKey); }
HoodieAvroWriteSupport extends AvroWriteSupport { public void add(String recordKey) { this.bloomFilter.add(recordKey); if (minRecordKey != null) { minRecordKey = minRecordKey.compareTo(recordKey) <= 0 ? minRecordKey : recordKey; } else { minRecordKey = recordKey; } if (maxRecordKey != null) { maxRecordKey = maxRecordKey.compareTo(recordKey) >= 0 ? maxRecordKey : recordKey; } else { maxRecordKey = recordKey; } } HoodieAvroWriteSupport(MessageType schema, Schema avroSchema, BloomFilter bloomFilter); @Override WriteSupport.FinalizedWriteContext finalizeWrite(); void add(String recordKey); static final String OLD_HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY; static final String HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY; static final String HOODIE_MIN_RECORD_KEY_FOOTER; static final String HOODIE_MAX_RECORD_KEY_FOOTER; static final String HOODIE_BLOOM_FILTER_TYPE_CODE; }