TerrapinUtil { static public List<Pair<Path, Long>> getS3FileList(AWSCredentials credentials, String s3Bucket, String s3KeyPrefix) { List<Pair<Path, Long>> fileSizePairList = Lists.newArrayListWithCapacity( Constants.MAX_ALLOWED_SHARDS); AmazonS3Client s3Client = new AmazonS3Client(credentials); String prefix = s3KeyPrefix; String marker = prefix; while (true) { boolean reachedEnd = false; ObjectListing listing = s3Client.listObjects(new ListObjectsRequest(). withBucketName(s3Bucket). withMarker(marker)); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); if (summaries.isEmpty()) { break; } for (S3ObjectSummary summary: summaries) { if (summary.getKey().startsWith(prefix)) { fileSizePairList.add(new ImmutablePair(new Path("s3n", s3Bucket, "/" + summary.getKey()), summary.getSize())); if (fileSizePairList.size() > Constants.MAX_ALLOWED_SHARDS) { throw new RuntimeException("Too many files " + fileSizePairList.size()); } } else { reachedEnd = true; break; } } if (reachedEnd) { break; } marker = summaries.get(summaries.size() - 1).getKey(); } return fileSizePairList; }  static String getHelixInstanceFromHDFSHost(String hdfsHostName); static PropertiesConfiguration readPropertiesExitOnFailure(String configFile); static Integer extractPartitionName(String fileName, PartitionerType partitioner); static String formatPartitionName(int partitionNumber); static String getPartitionName(ByteBuffer key,
                                        PartitionerType partitionerType,
                                        int numPartitions); static String hdfsDirToHelixResource(String hdfsDir); static String helixResourceToHdfsDir(String helixResource); static ZooKeeperClient getZooKeeperClient(String zkQuorum, int sessionTimeoutSeconds); static int getBucketSize(int numPartitions, boolean enableZkCompression); static String extractFileSetFromPath(String resource); static Pair<String, Integer> getBucketizedResourceAndPartitionNum(String helixPartition); static Pair<String, Integer> getNonBucketizedResourceAndPartitionNum(
      String helixPartition); static Pair<String, Integer> getResourceAndPartitionNum(String helixPartition); static String getViewPartitionName(String resource, int partition); static int getViewPartitionNumber(String viewPartitionName); static void compressIdealState(IdealState is); static String getZKQuorumFromConf(PropertiesConfiguration configuration); static List<HdfsFileStatus> getHdfsFileList(DFSClient hdfsClient,
                                                     String hdfsDir); static void loadFileSetData(ZooKeeperManager zkManager, FileSetInfo fsInfo, Options options); static List<Pair<Path, Long>> getS3FileList(AWSCredentials credentials,
      String s3Bucket, String s3KeyPrefix); static void setupConfiguration(Configuration conf,
                                        long dfsBlockSize,
                                        int dfsReplication);  }
@Test @PrepareForTest(TerrapinUtil.class) public void testGetS3FileList() throws Exception { AmazonS3Client s3Client = mock(AmazonS3Client.class); ObjectListing objectListing = mock(ObjectListing.class); S3ObjectSummary summary1 = new S3ObjectSummary(); S3ObjectSummary summary2 = new S3ObjectSummary(); S3ObjectSummary summary3 = new S3ObjectSummary(); summary1.setKey("/abc/123"); summary2.setKey("/abc/456"); summary3.setKey("/def/123"); summary1.setSize(32432); summary2.setSize(213423); summary3.setSize(2334); List<S3ObjectSummary> summaries = ImmutableList.of(summary1, summary2, summary3); whenNew(AmazonS3Client.class).withAnyArguments().thenReturn(s3Client); when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); when(objectListing.getObjectSummaries()).thenReturn(summaries); List<Pair<Path, Long>> results = TerrapinUtil.getS3FileList(new AnonymousAWSCredentials(), "bucket", "/abc"); assertEquals(2, results.size()); assertTrue(results.get(0).getLeft().toString().endsWith(summary1.getKey())); assertEquals(new Long(summary1.getSize()), results.get(0).getRight()); assertTrue(results.get(1).getLeft().toString().endsWith(summary2.getKey())); assertEquals(new Long(summary2.getSize()), results.get(1).getRight()); }