S3ChangeLogStore implements ChangeLogStore { void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("listObjects: Retrieving domains from {} with mod time > {}", s3BucketName, modTime); } ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName(s3BucketName)); String objectName; while (objectListing != null) { final List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); boolean listTruncated = objectListing.isTruncated(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("listObjects: retrieved {} objects, more objects available - {}", objectSummaries.size(), listTruncated); } for (S3ObjectSummary objectSummary : objectSummaries) { if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) { continue; } objectName = objectSummary.getKey(); if (objectName.charAt(0) == '.') { continue; } domains.add(objectName); } if (!listTruncated) { break; } objectListing = s3.listNextBatchOfObjects(objectListing); } } S3ChangeLogStore();  S3ChangeLogStore(String awsRegion); @Override boolean supportsFullRefresh(); @Override SignedDomain getLocalSignedDomain(String domainName); @SuppressWarnings("EmptyMethod") @Override void removeLocalDomain(String domainName); @SuppressWarnings("EmptyMethod") @Override void saveLocalDomain(String domainName, SignedDomain signedDomain); @Override List<String> getLocalDomainList(); boolean getAllSignedDomains(List<String> domains); @Override Set<String> getServerDomainList(); @Override SignedDomains getServerDomainModifiedList(); @Override SignedDomain getServerSignedDomain(String domainName); @Override SignedDomains getUpdatedSignedDomains(StringBuilder lastModTimeBuffer); @Override void setLastModificationTimestamp(String newLastModTime); ExecutorService getExecutorService();  }
@Test public void testListObjectsAllObjectsNoPage() { MockS3ChangeLogStore store = new MockS3ChangeLogStore(); ArrayList<S3ObjectSummary> objectList = new ArrayList<>(); S3ObjectSummary objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas"); objectList.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas.athenz"); objectList.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey(".date"); objectList.add(objectSummary); ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn(objectList); when(objectListing.isTruncated()).thenReturn(false); when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); ArrayList<String> domains = new ArrayList<>(); store.listObjects(store.awsS3Client, domains, 0); assertEquals(domains.size(), 2); assertTrue(domains.contains("iaas")); assertTrue(domains.contains("iaas.athenz")); }
@Test public void testListObjectsAllObjectsNoPageModTime() { MockS3ChangeLogStore store = new MockS3ChangeLogStore(); ArrayList<S3ObjectSummary> objectList = new ArrayList<>(); S3ObjectSummary objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas"); objectSummary.setLastModified(new Date(100)); objectList.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas.athenz"); objectSummary.setLastModified(new Date(200)); objectList.add(objectSummary); ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn(objectList); when(objectListing.isTruncated()).thenReturn(false); when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); ArrayList<String> domains = new ArrayList<>(); store.listObjects(store.awsS3Client, domains, (new Date(150)).getTime()); assertEquals(domains.size(), 1); assertTrue(domains.contains("iaas.athenz")); }
@Test public void testListObjectsAllObjectsMultiplePages() { MockS3ChangeLogStore store = new MockS3ChangeLogStore(); ArrayList<S3ObjectSummary> objectList1 = new ArrayList<>(); S3ObjectSummary objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas"); objectList1.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas.athenz"); objectList1.add(objectSummary); ArrayList<S3ObjectSummary> objectList2 = new ArrayList<>(); objectSummary = new S3ObjectSummary(); objectSummary.setKey("cd"); objectList2.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("cd.docker"); objectList2.add(objectSummary); ArrayList<S3ObjectSummary> objectList3 = new ArrayList<>(); objectSummary = new S3ObjectSummary(); objectSummary.setKey("platforms"); objectList3.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("platforms.mh2"); objectList3.add(objectSummary); ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()) .thenReturn(objectList1) .thenReturn(objectList2) .thenReturn(objectList3); when(objectListing.isTruncated()) .thenReturn(true) .thenReturn(true) .thenReturn(false); when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing); ArrayList<String> domains = new ArrayList<>(); store.listObjects(store.awsS3Client, domains, 0); assertEquals(domains.size(), 6); assertTrue(domains.contains("iaas")); assertTrue(domains.contains("iaas.athenz")); assertTrue(domains.contains("cd")); assertTrue(domains.contains("cd.docker")); assertTrue(domains.contains("platforms")); assertTrue(domains.contains("platforms.mh2")); }
@Test public void testListObjectsAllObjectsErrorCondition() { MockS3ChangeLogStore store = new MockS3ChangeLogStore(); ArrayList<S3ObjectSummary> objectList1 = new ArrayList<>(); S3ObjectSummary objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas"); objectList1.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas.athenz"); objectList1.add(objectSummary); ArrayList<S3ObjectSummary> objectList2 = new ArrayList<>(); objectSummary = new S3ObjectSummary(); objectSummary.setKey("cd"); objectList2.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("cd.docker"); objectList2.add(objectSummary); ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()) .thenReturn(objectList1) .thenReturn(objectList2); when(objectListing.isTruncated()) .thenReturn(true); when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))) .thenReturn(objectListing) .thenReturn(null); ArrayList<String> domains = new ArrayList<>(); store.listObjects(store.awsS3Client, domains, 0); assertEquals(domains.size(), 4); assertTrue(domains.contains("iaas")); assertTrue(domains.contains("iaas.athenz")); assertTrue(domains.contains("cd")); assertTrue(domains.contains("cd.docker")); }
@Test public void testListObjectsAllObjectsMultiplePagesModTime() { MockS3ChangeLogStore store = new MockS3ChangeLogStore(); ArrayList<S3ObjectSummary> objectList1 = new ArrayList<>(); S3ObjectSummary objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas"); objectSummary.setLastModified(new Date(100)); objectList1.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("iaas.athenz"); objectSummary.setLastModified(new Date(100)); objectList1.add(objectSummary); ArrayList<S3ObjectSummary> objectList2 = new ArrayList<>(); objectSummary = new S3ObjectSummary(); objectSummary.setKey("cd"); objectSummary.setLastModified(new Date(100)); objectList2.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("cd.docker"); objectSummary.setLastModified(new Date(200)); objectList2.add(objectSummary); ArrayList<S3ObjectSummary> objectList3 = new ArrayList<>(); objectSummary = new S3ObjectSummary(); objectSummary.setKey("platforms"); objectSummary.setLastModified(new Date(200)); objectList3.add(objectSummary); objectSummary = new S3ObjectSummary(); objectSummary.setKey("platforms.mh2"); objectSummary.setLastModified(new Date(200)); objectList3.add(objectSummary); ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()) .thenReturn(objectList1) .thenReturn(objectList2) .thenReturn(objectList3); when(objectListing.isTruncated()) .thenReturn(true) .thenReturn(true) .thenReturn(false); when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing); ArrayList<String> domains = new ArrayList<>(); store.listObjects(store.awsS3Client, domains, (new Date(150)).getTime()); assertEquals(domains.size(), 3); assertTrue(domains.contains("cd.docker")); assertTrue(domains.contains("platforms")); assertTrue(domains.contains("platforms.mh2")); }