@Test public void testOverwriteHoodieProperties() throws IOException { URL newProps = this.getClass().getClassLoader().getResource("table-config.properties"); assertNotNull(newProps, "New property file must exist"); CommandResult cr = getShell().executeCommand("repair overwrite-hoodie-props --new-props-file " + newProps.getPath()); assertTrue(cr.isSuccess()); Map<String, String> oldProps = HoodieCLI.getTableMetaClient().getTableConfig().getProps(); Map<String, String> result = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()).getTableConfig().getProps(); Properties expectProps = new Properties(); expectProps.load(new FileInputStream(new File(newProps.getPath()))); Map<String, String> expected = expectProps.entrySet().stream() .collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e -> String.valueOf(e.getValue()))); assertEquals(expected, result); List<String> allPropsStr = Arrays.asList("hoodie.table.name", "hoodie.table.type", "hoodie.table.version", "hoodie.archivelog.folder", "hoodie.timeline.layout.version"); String[][] rows = allPropsStr.stream().sorted().map(key -> new String[]{key, oldProps.getOrDefault(key, "null"), result.getOrDefault(key, "null")}) .toArray(String[][]::new); String expect = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY, HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows); expect = removeNonWordAndStripSpace(expect); String got = removeNonWordAndStripSpace(cr.getResult().toString()); assertEquals(expect, got); }
@CliCommand(value = "repair overwrite-hoodie-props", help = "Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!") public String overwriteHoodieProperties( @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with") final String overwriteFilePath) throws IOException { HoodieTableMetaClient client = HoodieCLI.getTableMetaClient(); Properties newProps = new Properties(); newProps.load(new FileInputStream(new File(overwriteFilePath))); Map<String, String> oldProps = client.getTableConfig().getProps(); Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME); HoodieTableConfig.createHoodieProperties(client.getFs(), metaPathDir, newProps); TreeSet<String> allPropKeys = new TreeSet<>(); allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet())); allPropKeys.addAll(oldProps.keySet()); String[][] rows = new String[allPropKeys.size()][]; int ind = 0; for (String propKey : allPropKeys) { String[] row = new String[]{ propKey, oldProps.getOrDefault(propKey, "null"), newProps.getOrDefault(propKey, "null").toString() }; rows[ind++] = row; } return HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY, HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows); }
RepairsCommand implements CommandMarker { @CliCommand(value = "repair overwrite-hoodie-props", help = "Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!") public String overwriteHoodieProperties( @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with") final String overwriteFilePath) throws IOException { HoodieTableMetaClient client = HoodieCLI.getTableMetaClient(); Properties newProps = new Properties(); newProps.load(new FileInputStream(new File(overwriteFilePath))); Map<String, String> oldProps = client.getTableConfig().getProps(); Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME); HoodieTableConfig.createHoodieProperties(client.getFs(), metaPathDir, newProps); TreeSet<String> allPropKeys = new TreeSet<>(); allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet())); allPropKeys.addAll(oldProps.keySet()); String[][] rows = new String[allPropKeys.size()][]; int ind = 0; for (String propKey : allPropKeys) { String[] row = new String[]{ propKey, oldProps.getOrDefault(propKey, "null"), newProps.getOrDefault(propKey, "null").toString() }; rows[ind++] = row; } return HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY, HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows); } }
RepairsCommand implements CommandMarker { @CliCommand(value = "repair overwrite-hoodie-props", help = "Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!") public String overwriteHoodieProperties( @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with") final String overwriteFilePath) throws IOException { HoodieTableMetaClient client = HoodieCLI.getTableMetaClient(); Properties newProps = new Properties(); newProps.load(new FileInputStream(new File(overwriteFilePath))); Map<String, String> oldProps = client.getTableConfig().getProps(); Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME); HoodieTableConfig.createHoodieProperties(client.getFs(), metaPathDir, newProps); TreeSet<String> allPropKeys = new TreeSet<>(); allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet())); allPropKeys.addAll(oldProps.keySet()); String[][] rows = new String[allPropKeys.size()][]; int ind = 0; for (String propKey : allPropKeys) { String[] row = new String[]{ propKey, oldProps.getOrDefault(propKey, "null"), newProps.getOrDefault(propKey, "null").toString() }; rows[ind++] = row; } return HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY, HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows); }  }
RepairsCommand implements CommandMarker { @CliCommand(value = "repair overwrite-hoodie-props", help = "Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!") public String overwriteHoodieProperties( @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with") final String overwriteFilePath) throws IOException { HoodieTableMetaClient client = HoodieCLI.getTableMetaClient(); Properties newProps = new Properties(); newProps.load(new FileInputStream(new File(overwriteFilePath))); Map<String, String> oldProps = client.getTableConfig().getProps(); Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME); HoodieTableConfig.createHoodieProperties(client.getFs(), metaPathDir, newProps); TreeSet<String> allPropKeys = new TreeSet<>(); allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet())); allPropKeys.addAll(oldProps.keySet()); String[][] rows = new String[allPropKeys.size()][]; int ind = 0; for (String propKey : allPropKeys) { String[] row = new String[]{ propKey, oldProps.getOrDefault(propKey, "null"), newProps.getOrDefault(propKey, "null").toString() }; rows[ind++] = row; } return HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY, HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows); }  @CliCommand(value = "repair deduplicate", help = "De-duplicate a partition path contains duplicates & produce repaired files to replace with") String deduplicate(
      @CliOption(key = {"duplicatedPartitionPath"}, help = "Partition Path containing the duplicates",
          mandatory = true) final String duplicatedPartitionPath,
      @CliOption(key = {"repairedOutputPath"}, help = "Location to place the repaired files",
          mandatory = true) final String repairedOutputPath,
      @CliOption(key = {"sparkProperties"}, help = "Spark Properties File Path",
          unspecifiedDefaultValue = "") String sparkPropertiesPath,
      @CliOption(key = "sparkMaster", unspecifiedDefaultValue = "", help = "Spark Master") String master,
      @CliOption(key = "sparkMemory", unspecifiedDefaultValue = "4G",
          help = "Spark executor memory") final String sparkMemory,
      @CliOption(key = {"dryrun"},
          help = "Should we actually remove duplicates or just run and store result to repairedOutputPath",
          unspecifiedDefaultValue = "true") final boolean dryRun,
      @CliOption(key = {"dedupeType"}, help = "Valid values are - insert_type, update_type and upsert_type",
          unspecifiedDefaultValue = "insert_type") final String dedupeType); @CliCommand(value = "repair addpartitionmeta", help = "Add partition metadata to a table, if not present") String addPartitionMeta(
      @CliOption(key = {"dryrun"}, help = "Should we actually add or just print what would be done",
          unspecifiedDefaultValue = "true") final boolean dryRun); @CliCommand(value = "repair overwrite-hoodie-props", help = "Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!") String overwriteHoodieProperties(
      @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with")
      final String overwriteFilePath); @CliCommand(value = "repair corrupted clean files", help = "repair corrupted clean files") void removeCorruptedPendingCleanAction(); }
RepairsCommand implements CommandMarker { @CliCommand(value = "repair overwrite-hoodie-props", help = "Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!") public String overwriteHoodieProperties( @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with") final String overwriteFilePath) throws IOException { HoodieTableMetaClient client = HoodieCLI.getTableMetaClient(); Properties newProps = new Properties(); newProps.load(new FileInputStream(new File(overwriteFilePath))); Map<String, String> oldProps = client.getTableConfig().getProps(); Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME); HoodieTableConfig.createHoodieProperties(client.getFs(), metaPathDir, newProps); TreeSet<String> allPropKeys = new TreeSet<>(); allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet())); allPropKeys.addAll(oldProps.keySet()); String[][] rows = new String[allPropKeys.size()][]; int ind = 0; for (String propKey : allPropKeys) { String[] row = new String[]{ propKey, oldProps.getOrDefault(propKey, "null"), newProps.getOrDefault(propKey, "null").toString() }; rows[ind++] = row; } return HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY, HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows); }  @CliCommand(value = "repair deduplicate", help = "De-duplicate a partition path contains duplicates & produce repaired files to replace with") String deduplicate(
      @CliOption(key = {"duplicatedPartitionPath"}, help = "Partition Path containing the duplicates",
          mandatory = true) final String duplicatedPartitionPath,
      @CliOption(key = {"repairedOutputPath"}, help = "Location to place the repaired files",
          mandatory = true) final String repairedOutputPath,
      @CliOption(key = {"sparkProperties"}, help = "Spark Properties File Path",
          unspecifiedDefaultValue = "") String sparkPropertiesPath,
      @CliOption(key = "sparkMaster", unspecifiedDefaultValue = "", help = "Spark Master") String master,
      @CliOption(key = "sparkMemory", unspecifiedDefaultValue = "4G",
          help = "Spark executor memory") final String sparkMemory,
      @CliOption(key = {"dryrun"},
          help = "Should we actually remove duplicates or just run and store result to repairedOutputPath",
          unspecifiedDefaultValue = "true") final boolean dryRun,
      @CliOption(key = {"dedupeType"}, help = "Valid values are - insert_type, update_type and upsert_type",
          unspecifiedDefaultValue = "insert_type") final String dedupeType); @CliCommand(value = "repair addpartitionmeta", help = "Add partition metadata to a table, if not present") String addPartitionMeta(
      @CliOption(key = {"dryrun"}, help = "Should we actually add or just print what would be done",
          unspecifiedDefaultValue = "true") final boolean dryRun); @CliCommand(value = "repair overwrite-hoodie-props", help = "Overwrite hoodie.properties with provided file. Risky operation. Proceed with caution!") String overwriteHoodieProperties(
      @CliOption(key = {"new-props-file"}, help = "Path to a properties file on local filesystem to overwrite the table's hoodie.properties with")
      final String overwriteFilePath); @CliCommand(value = "repair corrupted clean files", help = "repair corrupted clean files") void removeCorruptedPendingCleanAction(); static final String DEDUPLICATE_RETURN_PREFIX; }