test: fix CI flake by allowing retries on assertions against async actions (e.g. notifications)
MODIFY
gateway/test/src/test/java/io/apiman/gateway/test/junit/GatewayRestTester.java
gateway/test/src/test/java/io/apiman/gateway/test/junit/GatewayRestTester.java
@@ -234,14 +234,10 @@ public class GatewayRestTester extends ParentRunner<TestInfo> {
Integer delay = testInfo.test.getDelay();
if (delay != null) {
- try { Thread.sleep(delay); } catch (InterruptedException e) { }
+ try { Thread.sleep(delay); } catch (InterruptedException ignored) { }
}
if (rtPath != null && !rtPath.trim().isEmpty()) {
- RestTest restTest = TestUtil.loadRestTest(testInfo.test.getValue(), getTestClass().getJavaClass().getClassLoader());
- String endpoint = null;
- if (endpoint == null) {
- endpoint = testInfo.test.getEndpoint();
- }
+ String endpoint = testInfo.test.getEndpoint();
if (endpoint == null) {
endpoint = testInfo.group.getEndpoint();
}
@@ -254,14 +250,40 @@ public class GatewayRestTester extends ParentRunner<TestInfo> {
if (endpoint == null) {
endpoint = gatewayServer.getGatewayEndpoint();
}
-
- testInfo.plan.runner.runTest(restTest, endpoint);
+ runWithRetries(testInfo, endpoint);
gatewayServer.next(endpoint);
}
}
}, description, notifier);
}
+ private void runWithRetries(TestInfo testInfo, String endpoint) throws Throwable {
+ RestTest restTest = TestUtil.loadRestTest(testInfo.test.getValue(), getTestClass().getJavaClass().getClassLoader());
+ int maxRetries = testInfo.test.getMaxRetries();
+ int retryDelay = testInfo.test.getRetryDelay();
+ Throwable exHolder = null;
+
+ for (int tries = maxRetries; tries >= 0; tries--) {
+ if (maxRetries > 0) {
+ log("[Tries remaining: {0}]", tries);
+ }
+ try {
+ testInfo.plan.runner.runTest(restTest, endpoint);
+ return;
+ } catch (Throwable t) {
+ log("Error occurred: {0}", t.getMessage());
+ exHolder = t;
+ if (retryDelay > 0) {
+ log("Waiting {0} ms before retrying...", retryDelay);
+ Thread.sleep(retryDelay);
+ }
+ }
+ }
+ if (exHolder != null) {
+ throw exHolder;
+ }
+ }
+
/**
* Resolves the logical endpoint into a real endpoint provided by the {@link IGatewayTestServer}.
* @param endpoint

MODIFY
manager/test/api/src/main/java/io/apiman/manager/test/junit/ManagerRestTester.java
manager/test/api/src/main/java/io/apiman/manager/test/junit/ManagerRestTester.java
@@ -337,7 +337,7 @@ public class ManagerRestTester extends ParentRunner<TestInfo> {
if (StringUtils.isEmpty(endpoint)) {
endpoint = "http://localhost:" + getTestServerPort() + getBaseApiContext();
}
- testInfo.plan.runner.runTest(restTest, endpoint);
+ runWithRetries(testInfo, endpoint);
}
}
@@ -346,6 +346,33 @@ public class ManagerRestTester extends ParentRunner<TestInfo> {
}
}
+ private void runWithRetries(TestInfo testInfo, String endpoint) throws Throwable {
+ RestTest restTest = TestUtil.loadRestTest(testInfo.test.getValue(), getTestClass().getJavaClass().getClassLoader());
+ int maxRetries = testInfo.test.getMaxRetries();
+ int retryDelay = testInfo.test.getRetryDelay();
+ Throwable exHolder = null;
+
+ for (int tries = maxRetries; tries >= 0; tries--) {
+ if (maxRetries > 0) {
+ log("[Tries remaining: {0}]", tries);
+ }
+ try {
+ testInfo.plan.runner.runTest(restTest, endpoint);
+ return;
+ } catch (Throwable t) {
+ log("Error occurred: {0}", t.getMessage());
+ exHolder = t;
+ if (retryDelay > 0) {
+ log("Waiting {0} ms before retrying...", retryDelay);
+ Thread.sleep(retryDelay);
+ }
+ }
+ }
+ if (exHolder != null) {
+ throw exHolder;
+ }
+ }
+
/**
* @see org.junit.runners.ParentRunner#describeChild(java.lang.Object)
*/

MODIFY
manager/test/api/src/test/resources/test-plans/approvals-testPlan.xml
manager/test/api/src/test/resources/test-plans/approvals-testPlan.xml
<test name="Admin rejects contract 1 with client 2">test-plan-data/approvals/contracts/012_reject-contract-client2.resttest</test>
<test name="List Contracts for Client 2 (should be 0)">test-plan-data/approvals/contracts/013_get-contracts-client2-again.resttest</test>
- <test name="Approver receives notification about contract signup request" delay="5000">test-plan-data/approvals/notifications/001_approver-notifications.resttest</test>
- <test name="User bwayne receives notifications about approvals">test-plan-data/approvals/notifications/002_bwayne-notifications.resttest</test>
+ <test name="Approver receives notification about contract signup request" maxRetries="5">test-plan-data/approvals/notifications/001_approver-notifications.resttest</test>
+ <test name="User bwayne receives notifications about approvals" maxRetries="5">test-plan-data/approvals/notifications/002_bwayne-notifications.resttest</test>
<test name="bwayne mark all notifications read">test-plan-data/approvals/notifications/003_bwayne-mark-all-read.resttest</test>
- <test name="bwayne list notifications (all read)">test-plan-data/approvals/notifications/004_bwayne-list-notifications-all-read.resttest</test>
- <test name="bwayne list unread notifications (empty)">test-plan-data/approvals/notifications/005_bwayne-list-unread-notifications-empty.resttest</test>
+ <test name="bwayne list notifications (all read)" maxRetries="5">test-plan-data/approvals/notifications/004_bwayne-list-notifications-all-read.resttest</test>
+ <test name="bwayne list unread notifications (empty)" maxRetries="5">test-plan-data/approvals/notifications/005_bwayne-list-unread-notifications-empty.resttest</test>
- <test name="bwayne count all notifications">test-plan-data/approvals/notifications/006_bwayne-count-all-notifications.resttest</test>
- <test name="bwayne count unread notifications (zero)">test-plan-data/approvals/notifications/007_bwayne-count-unread-notifications.resttest</test>
+ <test name="bwayne count all notifications" maxRetries="5">test-plan-data/approvals/notifications/006_bwayne-count-all-notifications.resttest</test>
+ <test name="bwayne count unread notifications (zero)" maxRetries="5">test-plan-data/approvals/notifications/007_bwayne-count-unread-notifications.resttest</test>
</testGroup>
</testPlan>

MODIFY
test/common/src/main/java/io/apiman/test/common/plan/TestType.java
test/common/src/main/java/io/apiman/test/common/plan/TestType.java
package io.apiman.test.common.plan;
-import javax.xml.bind.annotation.*;
+import java.util.StringJoiner;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlType;
+import javax.xml.bind.annotation.XmlValue;
/**
@@ -46,6 +51,10 @@ public class TestType {
private Integer delay;
@XmlAttribute(name = "skipStorage")
private String skipStorage;
+ @XmlAttribute(name = "maxRetries")
+ private int maxRetries;
+ @XmlAttribute(name = "retryDelay")
+ private int retryDelay = 1_000;
/**
* Gets the value of the value property.
@@ -157,4 +166,34 @@ public class TestType {
this.delay = delay;
}
+ public int getMaxRetries() {
+ return maxRetries;
+ }
+
+ public TestType setMaxRetries(int maxRetries) {
+ this.maxRetries = maxRetries;
+ return this;
+ }
+
+ public int getRetryDelay() {
+ return retryDelay;
+ }
+
+ public TestType setRetryDelay(int retryDelay) {
+ this.retryDelay = retryDelay;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", TestType.class.getSimpleName() + "[", "]")
+ .add("value='" + value + "'")
+ .add("name='" + name + "'")
+ .add("endpoint='" + endpoint + "'")
+ .add("delay=" + delay)
+ .add("skipStorage='" + skipStorage + "'")
+ .add("maxRetries=" + maxRetries)
+ .add("retryDelay=" + retryDelay)
+ .toString();
+ }
}

MODIFY
test/common/src/main/resources/schemas/testPlan.xsd
test/common/src/main/resources/schemas/testPlan.xsd
<xsd:attribute name="endpoint" type="xsd:string" use="optional" />
<xsd:attribute name="delay" type="xsd:integer" use="optional" />
<xsd:attribute name="skipStorage" type="xsd:string" use="optional" />
+ <!--
+ For assertions against stuff that might be async such as notifications, let client retry a few times to let backend catch up.
+ Beware, this might have side effects if the endpoint is not idempotent.
+ -->
+ <xsd:attribute name="maxRetries" type="xsd:positiveInteger" default="0"/>
+ <xsd:attribute name="retryDelay" type="xsd:positiveInteger" default="1000"/>
</xsd:extension>
</xsd:simpleContent>
</xsd:complexType>

test: fix CI flake by allowing retries on assertions against async actions (e.g. notifications)