@Test public void testShowArchivedCommits() throws Exception { HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath) .withSchema(HoodieTestCommitMetadataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2) .withCompactionConfig(HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 3).build()) .forTable("test-trip-table").build(); Map<String, Integer[]> data = new LinkedHashMap<>(); data.put("104", new Integer[] {20, 10}); data.put("103", new Integer[] {15, 15}); data.put("102", new Integer[] {25, 45}); data.put("101", new Integer[] {35, 15}); for (Map.Entry<String, Integer[]> entry : data.entrySet()) { String key = entry.getKey(); Integer[] value = entry.getValue(); HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(), Option.of(value[0]), Option.of(value[1])); } metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()); HoodieSparkTable table = HoodieSparkTable.create(cfg, context, metaClient); HoodieTimelineArchiveLog archiveLog = new HoodieTimelineArchiveLog(cfg, table); archiveLog.archiveIfRequired(context); CommandResult cr = getShell().executeCommand(String.format("commits showarchived --startTs %s --endTs %s", "100", "104")); assertTrue(cr.isSuccess()); assertEquals(2, metaClient.reloadActiveTimeline().getCommitsTimeline().countInstants(), "There should 2 instants not be archived!"); data.remove("103"); data.remove("104"); String expected = generateExpectData(3, data); expected = removeNonWordAndStripSpace(expected); String got = removeNonWordAndStripSpace(cr.getResult().toString()); assertEquals(expected, got); }
@CliCommand(value = "commits showarchived", help = "Show the archived commits") public String showArchivedCommits( @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata", unspecifiedDefaultValue = "false") final boolean includeExtraMetadata, @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table", unspecifiedDefaultValue = "") final String exportTableName, @CliOption(key = {"startTs"}, mandatory = false, help = "start time for commits, default: now - 10 days") String startTs, @CliOption(key = {"endTs"}, mandatory = false, help = "end time for commits, default: now - 1 day") String endTs, @CliOption(key = {"limit"}, mandatory = false, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { if (StringUtils.isNullOrEmpty(startTs)) { startTs = CommitUtil.getTimeDaysAgo(10); } if (StringUtils.isNullOrEmpty(endTs)) { endTs = CommitUtil.getTimeDaysAgo(1); } HoodieArchivedTimeline archivedTimeline = HoodieCLI.getTableMetaClient().getArchivedTimeline(); try { archivedTimeline.loadInstantDetailsInMemory(startTs, endTs); HoodieDefaultTimeline timelineRange = archivedTimeline.findInstantsInRange(startTs, endTs); if (includeExtraMetadata) { return printCommitsWithMetadata(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } else { return printCommits(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } } finally { archivedTimeline.clearInstantDetailsFromMemory(startTs, endTs); } }
CommitsCommand implements CommandMarker { @CliCommand(value = "commits showarchived", help = "Show the archived commits") public String showArchivedCommits( @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata", unspecifiedDefaultValue = "false") final boolean includeExtraMetadata, @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table", unspecifiedDefaultValue = "") final String exportTableName, @CliOption(key = {"startTs"}, mandatory = false, help = "start time for commits, default: now - 10 days") String startTs, @CliOption(key = {"endTs"}, mandatory = false, help = "end time for commits, default: now - 1 day") String endTs, @CliOption(key = {"limit"}, mandatory = false, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { if (StringUtils.isNullOrEmpty(startTs)) { startTs = CommitUtil.getTimeDaysAgo(10); } if (StringUtils.isNullOrEmpty(endTs)) { endTs = CommitUtil.getTimeDaysAgo(1); } HoodieArchivedTimeline archivedTimeline = HoodieCLI.getTableMetaClient().getArchivedTimeline(); try { archivedTimeline.loadInstantDetailsInMemory(startTs, endTs); HoodieDefaultTimeline timelineRange = archivedTimeline.findInstantsInRange(startTs, endTs); if (includeExtraMetadata) { return printCommitsWithMetadata(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } else { return printCommits(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } } finally { archivedTimeline.clearInstantDetailsFromMemory(startTs, endTs); } } }
CommitsCommand implements CommandMarker { @CliCommand(value = "commits showarchived", help = "Show the archived commits") public String showArchivedCommits( @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata", unspecifiedDefaultValue = "false") final boolean includeExtraMetadata, @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table", unspecifiedDefaultValue = "") final String exportTableName, @CliOption(key = {"startTs"}, mandatory = false, help = "start time for commits, default: now - 10 days") String startTs, @CliOption(key = {"endTs"}, mandatory = false, help = "end time for commits, default: now - 1 day") String endTs, @CliOption(key = {"limit"}, mandatory = false, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { if (StringUtils.isNullOrEmpty(startTs)) { startTs = CommitUtil.getTimeDaysAgo(10); } if (StringUtils.isNullOrEmpty(endTs)) { endTs = CommitUtil.getTimeDaysAgo(1); } HoodieArchivedTimeline archivedTimeline = HoodieCLI.getTableMetaClient().getArchivedTimeline(); try { archivedTimeline.loadInstantDetailsInMemory(startTs, endTs); HoodieDefaultTimeline timelineRange = archivedTimeline.findInstantsInRange(startTs, endTs); if (includeExtraMetadata) { return printCommitsWithMetadata(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } else { return printCommits(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } } finally { archivedTimeline.clearInstantDetailsFromMemory(startTs, endTs); } }  }
CommitsCommand implements CommandMarker { @CliCommand(value = "commits showarchived", help = "Show the archived commits") public String showArchivedCommits( @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata", unspecifiedDefaultValue = "false") final boolean includeExtraMetadata, @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table", unspecifiedDefaultValue = "") final String exportTableName, @CliOption(key = {"startTs"}, mandatory = false, help = "start time for commits, default: now - 10 days") String startTs, @CliOption(key = {"endTs"}, mandatory = false, help = "end time for commits, default: now - 1 day") String endTs, @CliOption(key = {"limit"}, mandatory = false, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { if (StringUtils.isNullOrEmpty(startTs)) { startTs = CommitUtil.getTimeDaysAgo(10); } if (StringUtils.isNullOrEmpty(endTs)) { endTs = CommitUtil.getTimeDaysAgo(1); } HoodieArchivedTimeline archivedTimeline = HoodieCLI.getTableMetaClient().getArchivedTimeline(); try { archivedTimeline.loadInstantDetailsInMemory(startTs, endTs); HoodieDefaultTimeline timelineRange = archivedTimeline.findInstantsInRange(startTs, endTs); if (includeExtraMetadata) { return printCommitsWithMetadata(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } else { return printCommits(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } } finally { archivedTimeline.clearInstantDetailsFromMemory(startTs, endTs); } }  @CliCommand(value = "commits show", help = "Show the commits") String showCommits(
      @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata",
          unspecifiedDefaultValue = "false") final boolean includeExtraMetadata,
      @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table",
          unspecifiedDefaultValue = "") final String exportTableName,
      @CliOption(key = {"limit"}, help = "Limit commits",
          unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); @CliCommand(value = "commits showarchived", help = "Show the archived commits") String showArchivedCommits(
          @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata",
                  unspecifiedDefaultValue = "false") final boolean includeExtraMetadata,
          @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table",
                  unspecifiedDefaultValue = "") final String exportTableName,
          @CliOption(key = {"startTs"},  mandatory = false, help = "start time for commits, default: now - 10 days")
          String startTs,
          @CliOption(key = {"endTs"},  mandatory = false, help = "end time for commits, default: now - 1 day")
          String endTs,
          @CliOption(key = {"limit"}, mandatory = false, help = "Limit commits", unspecifiedDefaultValue = "-1")
          final Integer limit,
          @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "")
          final String sortByField,
          @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false")
          final boolean descending,
          @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false")
          final boolean headerOnly); @CliCommand(value = "commit rollback", help = "Rollback a commit") String rollbackCommit(@CliOption(key = {"commit"}, help = "Commit to rollback") final String instantTime,
      @CliOption(key = {"sparkProperties"}, help = "Spark Properties File Path") final String sparkPropertiesPath,
      @CliOption(key = "sparkMaster", unspecifiedDefaultValue = "", help = "Spark Master") String master,
      @CliOption(key = "sparkMemory", unspecifiedDefaultValue = "4G",
         help = "Spark executor memory") final String sparkMemory); @CliCommand(value = "commit showpartitions", help = "Show partition level details of a commit") String showCommitPartitions(
      @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table",
          unspecifiedDefaultValue = "") final String exportTableName,
      @CliOption(key = {"commit"}, help = "Commit to show") final String instantTime,
      @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); @CliCommand(value = "commit showfiles", help = "Show file level details of a commit") String showCommitFiles(
      @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table",
          unspecifiedDefaultValue = "") final String exportTableName,
      @CliOption(key = {"commit"}, help = "Commit to show") final String instantTime,
      @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); @CliCommand(value = "commits compare", help = "Compare commits with another Hoodie table") String compareCommits(@CliOption(key = {"path"}, help = "Path of the table to compare to") final String path); @CliCommand(value = "commits sync", help = "Compare commits with another Hoodie table") String syncCommits(@CliOption(key = {"path"}, help = "Path of the table to compare to") final String path); }
CommitsCommand implements CommandMarker { @CliCommand(value = "commits showarchived", help = "Show the archived commits") public String showArchivedCommits( @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata", unspecifiedDefaultValue = "false") final boolean includeExtraMetadata, @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table", unspecifiedDefaultValue = "") final String exportTableName, @CliOption(key = {"startTs"}, mandatory = false, help = "start time for commits, default: now - 10 days") String startTs, @CliOption(key = {"endTs"}, mandatory = false, help = "end time for commits, default: now - 1 day") String endTs, @CliOption(key = {"limit"}, mandatory = false, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { if (StringUtils.isNullOrEmpty(startTs)) { startTs = CommitUtil.getTimeDaysAgo(10); } if (StringUtils.isNullOrEmpty(endTs)) { endTs = CommitUtil.getTimeDaysAgo(1); } HoodieArchivedTimeline archivedTimeline = HoodieCLI.getTableMetaClient().getArchivedTimeline(); try { archivedTimeline.loadInstantDetailsInMemory(startTs, endTs); HoodieDefaultTimeline timelineRange = archivedTimeline.findInstantsInRange(startTs, endTs); if (includeExtraMetadata) { return printCommitsWithMetadata(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } else { return printCommits(timelineRange, limit, sortByField, descending, headerOnly, exportTableName); } } finally { archivedTimeline.clearInstantDetailsFromMemory(startTs, endTs); } }  @CliCommand(value = "commits show", help = "Show the commits") String showCommits(
      @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata",
          unspecifiedDefaultValue = "false") final boolean includeExtraMetadata,
      @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table",
          unspecifiedDefaultValue = "") final String exportTableName,
      @CliOption(key = {"limit"}, help = "Limit commits",
          unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); @CliCommand(value = "commits showarchived", help = "Show the archived commits") String showArchivedCommits(
          @CliOption(key = {"includeExtraMetadata"}, help = "Include extra metadata",
                  unspecifiedDefaultValue = "false") final boolean includeExtraMetadata,
          @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table",
                  unspecifiedDefaultValue = "") final String exportTableName,
          @CliOption(key = {"startTs"},  mandatory = false, help = "start time for commits, default: now - 10 days")
          String startTs,
          @CliOption(key = {"endTs"},  mandatory = false, help = "end time for commits, default: now - 1 day")
          String endTs,
          @CliOption(key = {"limit"}, mandatory = false, help = "Limit commits", unspecifiedDefaultValue = "-1")
          final Integer limit,
          @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "")
          final String sortByField,
          @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false")
          final boolean descending,
          @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false")
          final boolean headerOnly); @CliCommand(value = "commit rollback", help = "Rollback a commit") String rollbackCommit(@CliOption(key = {"commit"}, help = "Commit to rollback") final String instantTime,
      @CliOption(key = {"sparkProperties"}, help = "Spark Properties File Path") final String sparkPropertiesPath,
      @CliOption(key = "sparkMaster", unspecifiedDefaultValue = "", help = "Spark Master") String master,
      @CliOption(key = "sparkMemory", unspecifiedDefaultValue = "4G",
         help = "Spark executor memory") final String sparkMemory); @CliCommand(value = "commit showpartitions", help = "Show partition level details of a commit") String showCommitPartitions(
      @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table",
          unspecifiedDefaultValue = "") final String exportTableName,
      @CliOption(key = {"commit"}, help = "Commit to show") final String instantTime,
      @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); @CliCommand(value = "commit showfiles", help = "Show file level details of a commit") String showCommitFiles(
      @CliOption(key = {"createView"}, mandatory = false, help = "view name to store output table",
          unspecifiedDefaultValue = "") final String exportTableName,
      @CliOption(key = {"commit"}, help = "Commit to show") final String instantTime,
      @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); @CliCommand(value = "commits compare", help = "Compare commits with another Hoodie table") String compareCommits(@CliOption(key = {"path"}, help = "Path of the table to compare to") final String path); @CliCommand(value = "commits sync", help = "Compare commits with another Hoodie table") String syncCommits(@CliOption(key = {"path"}, help = "Path of the table to compare to") final String path);  }