FileBasedSource extends OffsetBasedSource<T> { @Override public final long getEstimatedSizeBytes(PipelineOptions options) throws IOException { String fileOrPattern = fileOrPatternSpec.get(); if (mode == Mode.FILEPATTERN) { long totalSize = 0; List<Metadata> allMatches = FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata(); for (Metadata metadata : allMatches) { totalSize += metadata.sizeBytes(); } LOG.info( "Filepattern {} matched {} files with total size {}", fileOrPattern, allMatches.size(), totalSize); return totalSize; } else { long start = getStartOffset(); long end = Math.min(getEndOffset(), getMaxEndOffset(options)); return end - start; } } protected  FileBasedSource(
      ValueProvider<String> fileOrPatternSpec,
      EmptyMatchTreatment emptyMatchTreatment,
      long minBundleSize); protected  FileBasedSource(ValueProvider<String> fileOrPatternSpec, long minBundleSize); protected  FileBasedSource(
      Metadata fileMetadata, long minBundleSize, long startOffset, long endOffset); final MatchResult.Metadata getSingleFileMetadata(); final String getFileOrPatternSpec(); final ValueProvider<String> getFileOrPatternSpecProvider(); final EmptyMatchTreatment getEmptyMatchTreatment(); final Mode getMode(); @Override final FileBasedSource<T> createSourceForSubrange(long start, long end); @Override final long getEstimatedSizeBytes(PipelineOptions options); @Override void populateDisplayData(DisplayData.Builder builder); @Override final List<? extends FileBasedSource<T>> split(
      long desiredBundleSizeBytes, PipelineOptions options); @Override final BoundedReader<T> createReader(PipelineOptions options); @Override String toString(); @Override void validate(); @Override final long getMaxEndOffset(PipelineOptions options);  }
@Test public void testEstimatedSizeOfFile() throws Exception { List<String> data = createStringDataset(3, 50); String fileName = "file"; File file = createFileWithData(fileName, data); TestFileBasedSource source = new TestFileBasedSource(file.getPath(), 64, null); assertEquals(file.length(), source.getEstimatedSizeBytes(null)); }
@Test public void testEstimatedSizeOfFilePattern() throws Exception { List<String> data1 = createStringDataset(3, 20); File file1 = createFileWithData("file1", data1); List<String> data2 = createStringDataset(3, 40); File file2 = createFileWithData("file2", data2); List<String> data3 = createStringDataset(3, 30); File file3 = createFileWithData("file3", data3); List<String> data4 = createStringDataset(3, 45); createFileWithData("otherfile", data4); List<String> data5 = createStringDataset(3, 53); createFileWithData("anotherfile", data5); TestFileBasedSource source = new TestFileBasedSource(new File(file1.getParent(), "file*").getPath(), 64, null); assertEquals( file1.length() + file2.length() + file3.length(), source.getEstimatedSizeBytes(null)); }