@Test public void testGetAllPendingCompactionOperationsWithFullDupFileId() throws IOException { HoodieCompactionPlan plan1 = createCompactionPlan(metaClient, "000", "001", 10, true, true); HoodieCompactionPlan plan2 = createCompactionPlan(metaClient, "002", "003", 0, false, false); scheduleCompaction(metaClient, "001", plan1); scheduleCompaction(metaClient, "003", plan2); scheduleCompaction(metaClient, "005", plan1); metaClient = new HoodieTableMetaClient(metaClient.getHadoopConf(), basePath, true); Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> res = CompactionUtils.getAllPendingCompactionOperations(metaClient); }
public static Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> getAllPendingCompactionOperations( HoodieTableMetaClient metaClient) { List<Pair<HoodieInstant, HoodieCompactionPlan>> pendingCompactionPlanWithInstants = getAllPendingCompactionPlans(metaClient); Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> fgIdToPendingCompactionWithInstantMap = new HashMap<>(); pendingCompactionPlanWithInstants.stream().flatMap(instantPlanPair -> getPendingCompactionOperations(instantPlanPair.getKey(), instantPlanPair.getValue())).forEach(pair -> { if (fgIdToPendingCompactionWithInstantMap.containsKey(pair.getKey())) { HoodieCompactionOperation operation = pair.getValue().getValue(); HoodieCompactionOperation anotherOperation = fgIdToPendingCompactionWithInstantMap.get(pair.getKey()).getValue(); if (!operation.equals(anotherOperation)) { String msg = "Hudi File Id (" + pair.getKey() + ") has more than 1 pending compactions. Instants: " + pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()); throw new IllegalStateException(msg); } } fgIdToPendingCompactionWithInstantMap.put(pair.getKey(), pair.getValue()); }); return fgIdToPendingCompactionWithInstantMap; }
CompactionUtils { public static Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> getAllPendingCompactionOperations( HoodieTableMetaClient metaClient) { List<Pair<HoodieInstant, HoodieCompactionPlan>> pendingCompactionPlanWithInstants = getAllPendingCompactionPlans(metaClient); Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> fgIdToPendingCompactionWithInstantMap = new HashMap<>(); pendingCompactionPlanWithInstants.stream().flatMap(instantPlanPair -> getPendingCompactionOperations(instantPlanPair.getKey(), instantPlanPair.getValue())).forEach(pair -> { if (fgIdToPendingCompactionWithInstantMap.containsKey(pair.getKey())) { HoodieCompactionOperation operation = pair.getValue().getValue(); HoodieCompactionOperation anotherOperation = fgIdToPendingCompactionWithInstantMap.get(pair.getKey()).getValue(); if (!operation.equals(anotherOperation)) { String msg = "Hudi File Id (" + pair.getKey() + ") has more than 1 pending compactions. Instants: " + pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()); throw new IllegalStateException(msg); } } fgIdToPendingCompactionWithInstantMap.put(pair.getKey(), pair.getValue()); }); return fgIdToPendingCompactionWithInstantMap; } }
CompactionUtils { public static Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> getAllPendingCompactionOperations( HoodieTableMetaClient metaClient) { List<Pair<HoodieInstant, HoodieCompactionPlan>> pendingCompactionPlanWithInstants = getAllPendingCompactionPlans(metaClient); Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> fgIdToPendingCompactionWithInstantMap = new HashMap<>(); pendingCompactionPlanWithInstants.stream().flatMap(instantPlanPair -> getPendingCompactionOperations(instantPlanPair.getKey(), instantPlanPair.getValue())).forEach(pair -> { if (fgIdToPendingCompactionWithInstantMap.containsKey(pair.getKey())) { HoodieCompactionOperation operation = pair.getValue().getValue(); HoodieCompactionOperation anotherOperation = fgIdToPendingCompactionWithInstantMap.get(pair.getKey()).getValue(); if (!operation.equals(anotherOperation)) { String msg = "Hudi File Id (" + pair.getKey() + ") has more than 1 pending compactions. Instants: " + pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()); throw new IllegalStateException(msg); } } fgIdToPendingCompactionWithInstantMap.put(pair.getKey(), pair.getValue()); }); return fgIdToPendingCompactionWithInstantMap; }  }
CompactionUtils { public static Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> getAllPendingCompactionOperations( HoodieTableMetaClient metaClient) { List<Pair<HoodieInstant, HoodieCompactionPlan>> pendingCompactionPlanWithInstants = getAllPendingCompactionPlans(metaClient); Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> fgIdToPendingCompactionWithInstantMap = new HashMap<>(); pendingCompactionPlanWithInstants.stream().flatMap(instantPlanPair -> getPendingCompactionOperations(instantPlanPair.getKey(), instantPlanPair.getValue())).forEach(pair -> { if (fgIdToPendingCompactionWithInstantMap.containsKey(pair.getKey())) { HoodieCompactionOperation operation = pair.getValue().getValue(); HoodieCompactionOperation anotherOperation = fgIdToPendingCompactionWithInstantMap.get(pair.getKey()).getValue(); if (!operation.equals(anotherOperation)) { String msg = "Hudi File Id (" + pair.getKey() + ") has more than 1 pending compactions. Instants: " + pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()); throw new IllegalStateException(msg); } } fgIdToPendingCompactionWithInstantMap.put(pair.getKey(), pair.getValue()); }); return fgIdToPendingCompactionWithInstantMap; }  static HoodieCompactionOperation buildFromFileSlice(String partitionPath, FileSlice fileSlice,
      Option<Function<Pair<String, FileSlice>, Map<String, Double>>> metricsCaptureFunction); static HoodieCompactionPlan buildFromFileSlices(List<Pair<String, FileSlice>> partitionFileSlicePairs,
      Option<Map<String, String>> extraMetadata,
      Option<Function<Pair<String, FileSlice>, Map<String, Double>>> metricsCaptureFunction); static HoodieCompactionOperation buildHoodieCompactionOperation(CompactionOperation op); static CompactionOperation buildCompactionOperation(HoodieCompactionOperation hc); static List<Pair<HoodieInstant, HoodieCompactionPlan>> getAllPendingCompactionPlans(
      HoodieTableMetaClient metaClient); static HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, String compactionInstant); static Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> getAllPendingCompactionOperations(
      HoodieTableMetaClient metaClient); static Stream<Pair<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>>> getPendingCompactionOperations(
      HoodieInstant instant, HoodieCompactionPlan compactionPlan); static List<HoodieInstant> getPendingCompactionInstantTimes(HoodieTableMetaClient metaClient); }
CompactionUtils { public static Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> getAllPendingCompactionOperations( HoodieTableMetaClient metaClient) { List<Pair<HoodieInstant, HoodieCompactionPlan>> pendingCompactionPlanWithInstants = getAllPendingCompactionPlans(metaClient); Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> fgIdToPendingCompactionWithInstantMap = new HashMap<>(); pendingCompactionPlanWithInstants.stream().flatMap(instantPlanPair -> getPendingCompactionOperations(instantPlanPair.getKey(), instantPlanPair.getValue())).forEach(pair -> { if (fgIdToPendingCompactionWithInstantMap.containsKey(pair.getKey())) { HoodieCompactionOperation operation = pair.getValue().getValue(); HoodieCompactionOperation anotherOperation = fgIdToPendingCompactionWithInstantMap.get(pair.getKey()).getValue(); if (!operation.equals(anotherOperation)) { String msg = "Hudi File Id (" + pair.getKey() + ") has more than 1 pending compactions. Instants: " + pair.getValue() + ", " + fgIdToPendingCompactionWithInstantMap.get(pair.getKey()); throw new IllegalStateException(msg); } } fgIdToPendingCompactionWithInstantMap.put(pair.getKey(), pair.getValue()); }); return fgIdToPendingCompactionWithInstantMap; }  static HoodieCompactionOperation buildFromFileSlice(String partitionPath, FileSlice fileSlice,
      Option<Function<Pair<String, FileSlice>, Map<String, Double>>> metricsCaptureFunction); static HoodieCompactionPlan buildFromFileSlices(List<Pair<String, FileSlice>> partitionFileSlicePairs,
      Option<Map<String, String>> extraMetadata,
      Option<Function<Pair<String, FileSlice>, Map<String, Double>>> metricsCaptureFunction); static HoodieCompactionOperation buildHoodieCompactionOperation(CompactionOperation op); static CompactionOperation buildCompactionOperation(HoodieCompactionOperation hc); static List<Pair<HoodieInstant, HoodieCompactionPlan>> getAllPendingCompactionPlans(
      HoodieTableMetaClient metaClient); static HoodieCompactionPlan getCompactionPlan(HoodieTableMetaClient metaClient, String compactionInstant); static Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> getAllPendingCompactionOperations(
      HoodieTableMetaClient metaClient); static Stream<Pair<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>>> getPendingCompactionOperations(
      HoodieInstant instant, HoodieCompactionPlan compactionPlan); static List<HoodieInstant> getPendingCompactionInstantTimes(HoodieTableMetaClient metaClient); static final Integer COMPACTION_METADATA_VERSION_1; static final Integer COMPACTION_METADATA_VERSION_2; static final Integer LATEST_COMPACTION_METADATA_VERSION; }