@Test public void testNoFilesDeletedWhenNoSnapshotsExpired() throws Exception { table.newFastAppend() .appendFile(FILE_A) .commit(); ExpireSnapshotsActionResult results = Actions.forTable(table).expireSnapshots().execute(); checkExpirationResults(0L, 0L, 0L, results); }
@Override public ExpireSnapshotsActionResult execute() { Dataset<Row> originalFiles = null; try { originalFiles = buildValidFileDF().persist(); originalFiles.count(); ExpireSnapshots expireSnaps = table.expireSnapshots().cleanExpiredFiles(false); for (final Long id : expireSnapshotIdValues) { expireSnaps = expireSnaps.expireSnapshotId(id); } if (expireOlderThanValue != null) { expireSnaps = expireSnaps.expireOlderThan(expireOlderThanValue); } if (retainLastValue != null) { expireSnaps = expireSnaps.retainLast(retainLastValue); } expireSnaps.commit(); Dataset<Row> validFiles = buildValidFileDF(); Dataset<Row> filesToDelete = originalFiles.except(validFiles); return deleteFiles(filesToDelete.toLocalIterator()); } finally { if (originalFiles != null) { originalFiles.unpersist(); } } }
ExpireSnapshotsAction extends BaseAction<ExpireSnapshotsActionResult> { @Override public ExpireSnapshotsActionResult execute() { Dataset<Row> originalFiles = null; try { originalFiles = buildValidFileDF().persist(); originalFiles.count(); ExpireSnapshots expireSnaps = table.expireSnapshots().cleanExpiredFiles(false); for (final Long id : expireSnapshotIdValues) { expireSnaps = expireSnaps.expireSnapshotId(id); } if (expireOlderThanValue != null) { expireSnaps = expireSnaps.expireOlderThan(expireOlderThanValue); } if (retainLastValue != null) { expireSnaps = expireSnaps.retainLast(retainLastValue); } expireSnaps.commit(); Dataset<Row> validFiles = buildValidFileDF(); Dataset<Row> filesToDelete = originalFiles.except(validFiles); return deleteFiles(filesToDelete.toLocalIterator()); } finally { if (originalFiles != null) { originalFiles.unpersist(); } } } }
ExpireSnapshotsAction extends BaseAction<ExpireSnapshotsActionResult> { @Override public ExpireSnapshotsActionResult execute() { Dataset<Row> originalFiles = null; try { originalFiles = buildValidFileDF().persist(); originalFiles.count(); ExpireSnapshots expireSnaps = table.expireSnapshots().cleanExpiredFiles(false); for (final Long id : expireSnapshotIdValues) { expireSnaps = expireSnaps.expireSnapshotId(id); } if (expireOlderThanValue != null) { expireSnaps = expireSnaps.expireOlderThan(expireOlderThanValue); } if (retainLastValue != null) { expireSnaps = expireSnaps.retainLast(retainLastValue); } expireSnaps.commit(); Dataset<Row> validFiles = buildValidFileDF(); Dataset<Row> filesToDelete = originalFiles.except(validFiles); return deleteFiles(filesToDelete.toLocalIterator()); } finally { if (originalFiles != null) { originalFiles.unpersist(); } } } ExpireSnapshotsAction(SparkSession spark, Table table); }
ExpireSnapshotsAction extends BaseAction<ExpireSnapshotsActionResult> { @Override public ExpireSnapshotsActionResult execute() { Dataset<Row> originalFiles = null; try { originalFiles = buildValidFileDF().persist(); originalFiles.count(); ExpireSnapshots expireSnaps = table.expireSnapshots().cleanExpiredFiles(false); for (final Long id : expireSnapshotIdValues) { expireSnaps = expireSnaps.expireSnapshotId(id); } if (expireOlderThanValue != null) { expireSnaps = expireSnaps.expireOlderThan(expireOlderThanValue); } if (retainLastValue != null) { expireSnaps = expireSnaps.retainLast(retainLastValue); } expireSnaps.commit(); Dataset<Row> validFiles = buildValidFileDF(); Dataset<Row> filesToDelete = originalFiles.except(validFiles); return deleteFiles(filesToDelete.toLocalIterator()); } finally { if (originalFiles != null) { originalFiles.unpersist(); } } } ExpireSnapshotsAction(SparkSession spark, Table table); ExpireSnapshotsAction executeDeleteWith(ExecutorService executorService); ExpireSnapshotsAction expireSnapshotId(long expireSnapshotId); ExpireSnapshotsAction expireOlderThan(long timestampMillis); ExpireSnapshotsAction retainLast(int numSnapshots); ExpireSnapshotsAction deleteWith(Consumer<String> newDeleteFunc); @Override ExpireSnapshotsActionResult execute(); }
ExpireSnapshotsAction extends BaseAction<ExpireSnapshotsActionResult> { @Override public ExpireSnapshotsActionResult execute() { Dataset<Row> originalFiles = null; try { originalFiles = buildValidFileDF().persist(); originalFiles.count(); ExpireSnapshots expireSnaps = table.expireSnapshots().cleanExpiredFiles(false); for (final Long id : expireSnapshotIdValues) { expireSnaps = expireSnaps.expireSnapshotId(id); } if (expireOlderThanValue != null) { expireSnaps = expireSnaps.expireOlderThan(expireOlderThanValue); } if (retainLastValue != null) { expireSnaps = expireSnaps.retainLast(retainLastValue); } expireSnaps.commit(); Dataset<Row> validFiles = buildValidFileDF(); Dataset<Row> filesToDelete = originalFiles.except(validFiles); return deleteFiles(filesToDelete.toLocalIterator()); } finally { if (originalFiles != null) { originalFiles.unpersist(); } } } ExpireSnapshotsAction(SparkSession spark, Table table); ExpireSnapshotsAction executeDeleteWith(ExecutorService executorService); ExpireSnapshotsAction expireSnapshotId(long expireSnapshotId); ExpireSnapshotsAction expireOlderThan(long timestampMillis); ExpireSnapshotsAction retainLast(int numSnapshots); ExpireSnapshotsAction deleteWith(Consumer<String> newDeleteFunc); @Override ExpireSnapshotsActionResult execute();  }