@Test public void testFileSizeStats() throws Exception { String commit1 = "100"; String commit2 = "101"; Map<String, Integer[]> data = new LinkedHashMap<>(); data.put(commit1, new Integer[] {100, 120, 150}); data.put(commit2, new Integer[] {200, 180, 250, 300}); String partition1 = HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; String partition2 = HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; String partition3 = HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH; HoodieTestTable testTable = HoodieTestTable.of(metaClient); Integer[] data1 = data.get(commit1); assertTrue(3 <= data1.length); testTable.addCommit(commit1) .withBaseFilesInPartition(partition1, data1[0]) .withBaseFilesInPartition(partition2, data1[1]) .withBaseFilesInPartition(partition3, data1[2]); Integer[] data2 = data.get(commit2); assertTrue(4 <= data2.length); testTable.addCommit(commit2) .withBaseFilesInPartition(partition1, data2[0]) .withBaseFilesInPartition(partition2, data2[1], data2[2]) .withBaseFilesInPartition(partition3, data2[3]); CommandResult cr = getShell().executeCommand("stats filesizes"); assertTrue(cr.isSuccess()); Histogram globalHistogram = new Histogram(new UniformReservoir(StatsCommand.MAX_FILES)); HashMap<String, Histogram> commitHistoMap = new HashMap<>(); data.forEach((k, v) -> { commitHistoMap.put(k, new Histogram(new UniformReservoir(StatsCommand.MAX_FILES))); for (int value : v) { commitHistoMap.get(k).update(value); globalHistogram.update(value); } }); List<Comparable[]> rows = new ArrayList<>(); for (Map.Entry<String, Histogram> entry : commitHistoMap.entrySet()) { Snapshot s = entry.getValue().getSnapshot(); rows.add(new StatsCommand().printFileSizeHistogram(entry.getKey(), s)); } Snapshot s = globalHistogram.getSnapshot(); rows.add(new StatsCommand().printFileSizeHistogram("ALL", s)); TableHeader header = new TableHeader() .addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MIN) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_10TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_50TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_AVG) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_95TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MAX) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_NUM_FILES) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV); String expect = HoodiePrintHelper.print(header, new StatsCommand().getFieldNameToConverterMap(), "", false, -1, false, rows); expect = removeNonWordAndStripSpace(expect); String got = removeNonWordAndStripSpace(cr.getResult().toString()); assertEquals(expect, got); }
@CliCommand(value = "stats filesizes", help = "File Sizes. Display summary stats on sizes of files") public String fileSizeStats( @CliOption(key = {"partitionPath"}, help = "regex to select files, eg: 2016/08/02", unspecifiedDefaultValue = "*/*/*") final String globRegex, @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { FileSystem fs = HoodieCLI.fs; String globPath = String.format("%s/%s/*", HoodieCLI.getTableMetaClient().getBasePath(), globRegex); List<FileStatus> statuses = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(globPath)); Histogram globalHistogram = new Histogram(new UniformReservoir(MAX_FILES)); HashMap<String, Histogram> commitHistoMap = new HashMap<>(); for (FileStatus fileStatus : statuses) { String instantTime = FSUtils.getCommitTime(fileStatus.getPath().getName()); long sz = fileStatus.getLen(); if (!commitHistoMap.containsKey(instantTime)) { commitHistoMap.put(instantTime, new Histogram(new UniformReservoir(MAX_FILES))); } commitHistoMap.get(instantTime).update(sz); globalHistogram.update(sz); } List<Comparable[]> rows = new ArrayList<>(); for (Map.Entry<String, Histogram> entry : commitHistoMap.entrySet()) { Snapshot s = entry.getValue().getSnapshot(); rows.add(printFileSizeHistogram(entry.getKey(), s)); } Snapshot s = globalHistogram.getSnapshot(); rows.add(printFileSizeHistogram("ALL", s)); TableHeader header = new TableHeader() .addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MIN) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_10TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_50TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_AVG) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_95TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MAX) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_NUM_FILES) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV); return HoodiePrintHelper.print(header, getFieldNameToConverterMap(), sortByField, descending, limit, headerOnly, rows); }
StatsCommand implements CommandMarker { @CliCommand(value = "stats filesizes", help = "File Sizes. Display summary stats on sizes of files") public String fileSizeStats( @CliOption(key = {"partitionPath"}, help = "regex to select files, eg: 2016/08/02", unspecifiedDefaultValue = "*/*/*") final String globRegex, @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { FileSystem fs = HoodieCLI.fs; String globPath = String.format("%s/%s/*", HoodieCLI.getTableMetaClient().getBasePath(), globRegex); List<FileStatus> statuses = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(globPath)); Histogram globalHistogram = new Histogram(new UniformReservoir(MAX_FILES)); HashMap<String, Histogram> commitHistoMap = new HashMap<>(); for (FileStatus fileStatus : statuses) { String instantTime = FSUtils.getCommitTime(fileStatus.getPath().getName()); long sz = fileStatus.getLen(); if (!commitHistoMap.containsKey(instantTime)) { commitHistoMap.put(instantTime, new Histogram(new UniformReservoir(MAX_FILES))); } commitHistoMap.get(instantTime).update(sz); globalHistogram.update(sz); } List<Comparable[]> rows = new ArrayList<>(); for (Map.Entry<String, Histogram> entry : commitHistoMap.entrySet()) { Snapshot s = entry.getValue().getSnapshot(); rows.add(printFileSizeHistogram(entry.getKey(), s)); } Snapshot s = globalHistogram.getSnapshot(); rows.add(printFileSizeHistogram("ALL", s)); TableHeader header = new TableHeader() .addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MIN) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_10TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_50TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_AVG) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_95TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MAX) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_NUM_FILES) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV); return HoodiePrintHelper.print(header, getFieldNameToConverterMap(), sortByField, descending, limit, headerOnly, rows); } }
StatsCommand implements CommandMarker { @CliCommand(value = "stats filesizes", help = "File Sizes. Display summary stats on sizes of files") public String fileSizeStats( @CliOption(key = {"partitionPath"}, help = "regex to select files, eg: 2016/08/02", unspecifiedDefaultValue = "*/*/*") final String globRegex, @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { FileSystem fs = HoodieCLI.fs; String globPath = String.format("%s/%s/*", HoodieCLI.getTableMetaClient().getBasePath(), globRegex); List<FileStatus> statuses = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(globPath)); Histogram globalHistogram = new Histogram(new UniformReservoir(MAX_FILES)); HashMap<String, Histogram> commitHistoMap = new HashMap<>(); for (FileStatus fileStatus : statuses) { String instantTime = FSUtils.getCommitTime(fileStatus.getPath().getName()); long sz = fileStatus.getLen(); if (!commitHistoMap.containsKey(instantTime)) { commitHistoMap.put(instantTime, new Histogram(new UniformReservoir(MAX_FILES))); } commitHistoMap.get(instantTime).update(sz); globalHistogram.update(sz); } List<Comparable[]> rows = new ArrayList<>(); for (Map.Entry<String, Histogram> entry : commitHistoMap.entrySet()) { Snapshot s = entry.getValue().getSnapshot(); rows.add(printFileSizeHistogram(entry.getKey(), s)); } Snapshot s = globalHistogram.getSnapshot(); rows.add(printFileSizeHistogram("ALL", s)); TableHeader header = new TableHeader() .addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MIN) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_10TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_50TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_AVG) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_95TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MAX) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_NUM_FILES) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV); return HoodiePrintHelper.print(header, getFieldNameToConverterMap(), sortByField, descending, limit, headerOnly, rows); }  }
StatsCommand implements CommandMarker { @CliCommand(value = "stats filesizes", help = "File Sizes. Display summary stats on sizes of files") public String fileSizeStats( @CliOption(key = {"partitionPath"}, help = "regex to select files, eg: 2016/08/02", unspecifiedDefaultValue = "*/*/*") final String globRegex, @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { FileSystem fs = HoodieCLI.fs; String globPath = String.format("%s/%s/*", HoodieCLI.getTableMetaClient().getBasePath(), globRegex); List<FileStatus> statuses = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(globPath)); Histogram globalHistogram = new Histogram(new UniformReservoir(MAX_FILES)); HashMap<String, Histogram> commitHistoMap = new HashMap<>(); for (FileStatus fileStatus : statuses) { String instantTime = FSUtils.getCommitTime(fileStatus.getPath().getName()); long sz = fileStatus.getLen(); if (!commitHistoMap.containsKey(instantTime)) { commitHistoMap.put(instantTime, new Histogram(new UniformReservoir(MAX_FILES))); } commitHistoMap.get(instantTime).update(sz); globalHistogram.update(sz); } List<Comparable[]> rows = new ArrayList<>(); for (Map.Entry<String, Histogram> entry : commitHistoMap.entrySet()) { Snapshot s = entry.getValue().getSnapshot(); rows.add(printFileSizeHistogram(entry.getKey(), s)); } Snapshot s = globalHistogram.getSnapshot(); rows.add(printFileSizeHistogram("ALL", s)); TableHeader header = new TableHeader() .addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MIN) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_10TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_50TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_AVG) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_95TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MAX) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_NUM_FILES) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV); return HoodiePrintHelper.print(header, getFieldNameToConverterMap(), sortByField, descending, limit, headerOnly, rows); }  @CliCommand(value = "stats wa", help = "Write Amplification. Ratio of how many records were upserted to how many " + "records were actually written") String writeAmplificationStats(
      @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); Comparable[] printFileSizeHistogram(String instantTime, Snapshot s); @CliCommand(value = "stats filesizes", help = "File Sizes. Display summary stats on sizes of files") String fileSizeStats(
      @CliOption(key = {"partitionPath"}, help = "regex to select files, eg: 2016/08/02",
          unspecifiedDefaultValue = "*/*/*") final String globRegex,
      @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); Map<String, Function<Object, String>> getFieldNameToConverterMap(); }
StatsCommand implements CommandMarker { @CliCommand(value = "stats filesizes", help = "File Sizes. Display summary stats on sizes of files") public String fileSizeStats( @CliOption(key = {"partitionPath"}, help = "regex to select files, eg: 2016/08/02", unspecifiedDefaultValue = "*/*/*") final String globRegex, @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit, @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField, @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending, @CliOption(key = {"headeronly"}, help = "Print Header Only", unspecifiedDefaultValue = "false") final boolean headerOnly) throws IOException { FileSystem fs = HoodieCLI.fs; String globPath = String.format("%s/%s/*", HoodieCLI.getTableMetaClient().getBasePath(), globRegex); List<FileStatus> statuses = FSUtils.getGlobStatusExcludingMetaFolder(fs, new Path(globPath)); Histogram globalHistogram = new Histogram(new UniformReservoir(MAX_FILES)); HashMap<String, Histogram> commitHistoMap = new HashMap<>(); for (FileStatus fileStatus : statuses) { String instantTime = FSUtils.getCommitTime(fileStatus.getPath().getName()); long sz = fileStatus.getLen(); if (!commitHistoMap.containsKey(instantTime)) { commitHistoMap.put(instantTime, new Histogram(new UniformReservoir(MAX_FILES))); } commitHistoMap.get(instantTime).update(sz); globalHistogram.update(sz); } List<Comparable[]> rows = new ArrayList<>(); for (Map.Entry<String, Histogram> entry : commitHistoMap.entrySet()) { Snapshot s = entry.getValue().getSnapshot(); rows.add(printFileSizeHistogram(entry.getKey(), s)); } Snapshot s = globalHistogram.getSnapshot(); rows.add(printFileSizeHistogram("ALL", s)); TableHeader header = new TableHeader() .addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MIN) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_10TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_50TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_AVG) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_95TH) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MAX) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_NUM_FILES) .addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV); return HoodiePrintHelper.print(header, getFieldNameToConverterMap(), sortByField, descending, limit, headerOnly, rows); }  @CliCommand(value = "stats wa", help = "Write Amplification. Ratio of how many records were upserted to how many " + "records were actually written") String writeAmplificationStats(
      @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); Comparable[] printFileSizeHistogram(String instantTime, Snapshot s); @CliCommand(value = "stats filesizes", help = "File Sizes. Display summary stats on sizes of files") String fileSizeStats(
      @CliOption(key = {"partitionPath"}, help = "regex to select files, eg: 2016/08/02",
          unspecifiedDefaultValue = "*/*/*") final String globRegex,
      @CliOption(key = {"limit"}, help = "Limit commits", unspecifiedDefaultValue = "-1") final Integer limit,
      @CliOption(key = {"sortBy"}, help = "Sorting Field", unspecifiedDefaultValue = "") final String sortByField,
      @CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
      @CliOption(key = {"headeronly"}, help = "Print Header Only",
          unspecifiedDefaultValue = "false") final boolean headerOnly); Map<String, Function<Object, String>> getFieldNameToConverterMap(); static final int MAX_FILES; }