@Test public void testSizeEstimator() throws IOException, URISyntaxException { Schema schema = SchemaTestUtil.getSimpleSchema(); List<HoodieRecord> hoodieRecords = SchemaTestUtil.generateHoodieTestRecords(0, 1, schema); long payloadSize = SpillableMapUtils.computePayloadSize(hoodieRecords.remove(0), new HoodieRecordSizeEstimator(schema)); assertTrue(payloadSize > 0); schema = HoodieAvroUtils.addMetadataFields(schema); hoodieRecords = SchemaTestUtil.generateHoodieTestRecords(0, 1, schema); payloadSize = SpillableMapUtils.computePayloadSize(hoodieRecords.remove(0), new HoodieRecordSizeEstimator(schema)); assertTrue(payloadSize > 0); schema = SchemaTestUtil.getSimpleSchema(); List<IndexedRecord> indexedRecords = SchemaTestUtil.generateHoodieTestRecords(0, 1); hoodieRecords = indexedRecords.stream().map(r -> new HoodieRecord<>(new HoodieKey(UUID.randomUUID().toString(), "0000/00/00"), new AvroBinaryTestPayload(Option.of((GenericRecord) r)))).collect(Collectors.toList()); payloadSize = SpillableMapUtils.computePayloadSize(hoodieRecords.remove(0), new HoodieRecordSizeEstimator(schema)); assertTrue(payloadSize > 0); final Schema simpleSchemaWithMetadata = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema()); indexedRecords = SchemaTestUtil.generateHoodieTestRecords(0, 1); hoodieRecords = indexedRecords.stream() .map(r -> new HoodieRecord<>(new HoodieKey(UUID.randomUUID().toString(), "0000/00/00"), new AvroBinaryTestPayload( Option.of(HoodieAvroUtils.rewriteRecord((GenericRecord) r, simpleSchemaWithMetadata))))) .collect(Collectors.toList()); payloadSize = SpillableMapUtils.computePayloadSize(hoodieRecords.remove(0), new HoodieRecordSizeEstimator(schema)); assertTrue(payloadSize > 0); }
@Override public R remove(Object key) { R value = get(key); valueMetadataMap.remove(key); return value; }
DiskBasedMap implements Map<T, R>, Iterable<R> { @Override public R remove(Object key) { R value = get(key); valueMetadataMap.remove(key); return value; } }
DiskBasedMap implements Map<T, R>, Iterable<R> { @Override public R remove(Object key) { R value = get(key); valueMetadataMap.remove(key); return value; } DiskBasedMap(String baseFilePath); }
DiskBasedMap implements Map<T, R>, Iterable<R> { @Override public R remove(Object key) { R value = get(key); valueMetadataMap.remove(key); return value; } DiskBasedMap(String baseFilePath); @Override Iterator<R> iterator(); long sizeOfFileOnDiskInBytes(); @Override int size(); @Override boolean isEmpty(); @Override boolean containsKey(Object key); @Override boolean containsValue(Object value); @Override R get(Object key); static R get(ValueMetadata entry, RandomAccessFile file); @Override R put(T key, R value); @Override R remove(Object key); @Override void putAll(Map<? extends T, ? extends R> m); @Override void clear(); @Override Set<T> keySet(); @Override Collection<R> values(); Stream<R> valueStream(); @Override Set<Entry<T, R>> entrySet(); }
DiskBasedMap implements Map<T, R>, Iterable<R> { @Override public R remove(Object key) { R value = get(key); valueMetadataMap.remove(key); return value; } DiskBasedMap(String baseFilePath); @Override Iterator<R> iterator(); long sizeOfFileOnDiskInBytes(); @Override int size(); @Override boolean isEmpty(); @Override boolean containsKey(Object key); @Override boolean containsValue(Object value); @Override R get(Object key); static R get(ValueMetadata entry, RandomAccessFile file); @Override R put(T key, R value); @Override R remove(Object key); @Override void putAll(Map<? extends T, ? extends R> m); @Override void clear(); @Override Set<T> keySet(); @Override Collection<R> values(); Stream<R> valueStream(); @Override Set<Entry<T, R>> entrySet(); static final int BUFFER_SIZE; }