InternalStepRunner implements StepRunner { @Override public Optional<RunStatus> run(LockedStep step, RunId id) { DualLogger logger = new DualLogger(id, step.get()); try { switch (step.get()) { case deployTester: return deployTester(id, logger); case deployInitialReal: return deployInitialReal(id, logger); case installInitialReal: return installInitialReal(id, logger); case deployReal: return deployReal(id, logger); case installTester: return installTester(id, logger); case installReal: return installReal(id, logger); case startStagingSetup: return startTests(id, true, logger); case endStagingSetup: case endTests: return endTests(id, logger); case startTests: return startTests(id, false, logger); case copyVespaLogs: return copyVespaLogs(id, logger); case deactivateReal: return deactivateReal(id, logger); case deactivateTester: return deactivateTester(id, logger); case report: return report(id, logger); default: throw new AssertionError("Unknown step '" + step + "'!"); } } catch (UncheckedIOException e) { logger.logWithInternalException(INFO, "IO exception running " + id + ": " + Exceptions.toMessageString(e), e); return Optional.empty(); } catch (RuntimeException e) { logger.log(WARNING, "Unexpected exception running " + id, e); if (step.get().alwaysRun()) { logger.log("Will keep trying, as this is a cleanup step."); return Optional.empty(); } return Optional.of(error); } } InternalStepRunner(Controller controller); @Override Optional<RunStatus> run(LockedStep step, RunId id);  }
@Test public void restartsServicesAndWaitsForRestartAndReboot() { RunId id = app.newRun(JobType.productionUsCentral1); ZoneId zone = id.type().zone(system()); HostName host = tester.configServer().hostFor(instanceId, zone); tester.configServer().setConfigChangeActions(new ConfigChangeActions(singletonList(new RestartAction("cluster", "container", "search", singletonList(new ServiceInfo("queries", "search", "config", host.value())), singletonList("Restart it!"))), Collections.emptyList())); tester.runner().run(); assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal)); tester.configServer().convergeServices(app.instanceId(), zone); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); tester.configServer().nodeRepository().doRestart(app.deploymentIdIn(zone), Optional.of(host)); tester.configServer().nodeRepository().requestReboot(app.deploymentIdIn(zone), Optional.of(host)); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().plus(Duration.ofSeconds(1))); tester.runner().run(); assertEquals(installationFailed, tester.jobs().run(id).get().status()); }
@Test public void waitsForEndpointsAndTimesOut() { app.newRun(JobType.systemTest); var testZone = JobType.systemTest.zone(system()); var stagingZone = JobType.stagingTest.zone(system()); tester.newDeploymentContext(app.testerId().id()) .deferLoadBalancerProvisioningIn(testZone.environment()); tester.newDeploymentContext(app.instanceId()) .deferLoadBalancerProvisioningIn(stagingZone.environment()); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); tester.runner().run(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).endpoint().plus(Duration.ofSeconds(1))); tester.runner().run(); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); }
@Test public void timesOutWithoutInstallationProgress() { tester.controllerTester().upgradeSystem(new Version("7.1")); tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); app.newRun(JobType.systemTest); tester.runner().run(); tester.configServer().setVersion(tester.controller().systemVersion(), app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().setVersion(tester.controller().systemVersion(), app.testerId().id(), JobType.stagingTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), app.instanceId()).iterator().next(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); tester.configServer().nodeRepository().putNodes(JobType.systemTest.zone(system()), new Node.Builder(systemTestNode) .serviceState(Node.ServiceState.allowedDown) .suspendedSince(tester.clock().instant()) .build()); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); tester.clock().advance(Duration.ofSeconds(2)); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).nodesDown().minus(Duration.ofSeconds(3))); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); tester.clock().advance(Duration.ofSeconds(2)); tester.runner().run(); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); }
@Test public void startingTestsFailsIfDeploymentExpires() { app.newRun(JobType.systemTest); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.startTests)); assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasEnded()); assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasFailed()); }
@Test public void alternativeEndpointsAreDetected() { var systemTestZone = JobType.systemTest.zone(system()); var stagingZone = JobType.stagingTest.zone(system()); tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone)); var applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) .upgradePolicy("default") .region("us-central-1") .parallel("us-west-1", "us-east-3") .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) .build(); app.submit(applicationPackage) .triggerJobs(); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); app.flushDnsUpdates(); tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); }
@Test public void testsFailIfTesterRestarts() { RunId id = app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.NOT_STARTED); tester.runner().run(); assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); }
@Test public void testsFailIfTestsFailRemotely() { RunId id = app.startSystemTestTests(); tester.cloud().add(new LogEntry(123, Instant.ofEpochMilli(321), error, "Failure!")); tester.cloud().set(TesterCloud.Status.FAILURE); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.runner().run(); assertTestLogEntries(id, Step.endTests, new LogEntry(lastId + 1, Instant.ofEpochMilli(321), error, "Failure!"), new LogEntry(lastId + 2, tester.clock().instant(), info, "Tests failed.")); assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); }
@Test public void testsFailIfTestsErr() { RunId id = app.startSystemTestTests(); tester.cloud().add(new LogEntry(0, Instant.ofEpochMilli(123), error, "Error!")); tester.cloud().set(TesterCloud.Status.ERROR); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.runner().run(); assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); assertTestLogEntries(id, Step.endTests, new LogEntry(lastId + 1, Instant.ofEpochMilli(123), error, "Error!"), new LogEntry(lastId + 2, tester.clock().instant(), info, "Tester failed running its tests!")); }
@Test public void testsSucceedWhenTheyDoRemotely() { RunId id = app.startSystemTestTests(); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); var testZone = JobType.systemTest.zone(system()); Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); assertEquals(app.instanceId().serializedForm(), configObject.field("application").asString()); assertEquals(testZone.value(), configObject.field("zone").asString()); assertEquals(system().value(), configObject.field("system").asString()); assertEquals(1, configObject.field("zoneEndpoints").children()); assertEquals(1, configObject.field("zoneEndpoints").field(testZone.value()).children()); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.cloud().add(new LogEntry(0, Instant.ofEpochMilli(123), info, "Ready!")); tester.runner().run(); assertTestLogEntries(id, Step.endTests, new LogEntry(lastId + 1, Instant.ofEpochMilli(123), info, "Ready!")); tester.cloud().add(new LogEntry(1, Instant.ofEpochMilli(1234), info, "Steady!")); tester.runner().run(); assertTestLogEntries(id, Step.endTests, new LogEntry(lastId + 1, Instant.ofEpochMilli(123), info, "Ready!"), new LogEntry(lastId + 2, Instant.ofEpochMilli(1234), info, "Steady!")); tester.cloud().add(new LogEntry(12, Instant.ofEpochMilli(12345), info, "Success!")); tester.cloud().set(TesterCloud.Status.SUCCESS); tester.runner().run(); assertTestLogEntries(id, Step.endTests, new LogEntry(lastId + 1, Instant.ofEpochMilli(123), info, "Ready!"), new LogEntry(lastId + 2, Instant.ofEpochMilli(1234), info, "Steady!"), new LogEntry(lastId + 3, Instant.ofEpochMilli(12345), info, "Success!"), new LogEntry(lastId + 4, tester.clock().instant(), info, "Tests completed successfully.")); assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); }
@Test public void notificationIsSent() { app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.NOT_STARTED); tester.runner().run(); MockMailer mailer = tester.controllerTester().serviceRegistry().mailer(); assertEquals(1, mailer.inbox("a@b").size()); assertEquals("Vespa application tenant.application: System test failing due to system error", mailer.inbox("a@b").get(0).subject()); assertEquals(1, mailer.inbox("b@a").size()); assertEquals("Vespa application tenant.application: System test failing due to system error", mailer.inbox("b@a").get(0).subject()); }
@Test public void vespaLogIsCopied() { RunId id = app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.ERROR); tester.configServer().setLogStream(vespaLog); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.runner().run(); assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); assertTestLogEntries(id, Step.copyVespaLogs, new LogEntry(lastId + 2, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), new LogEntry(lastId + 3, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), new LogEntry(lastId + 4, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" + "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)")); }