@Test public void testRollingUpdateFailed() throws Exception { when(client.rollingUpdate(anyString(), any(JobId.class), any(RolloutOptions.class))) .thenReturn(immediateFuture(new RollingUpdateResponse(RollingUpdateResponse.Status.OK))); when(client.deploymentGroupStatus(GROUP_NAME)).then(new ResponseAnswer( statusResponse(DeploymentGroupStatusResponse.Status.ROLLING_OUT, null, makeHostStatus("host1", JOB_ID, TaskStatus.State.PULLING_IMAGE), makeHostStatus("host2", null, null)), statusResponse(DeploymentGroupStatusResponse.Status.FAILED, "foobar", makeHostStatus("host1", JOB_ID, TaskStatus.State.RUNNING), makeHostStatus("host2", null, null)) )); final int ret = command.runWithJob(options, client, out, false, JOB, null); final String output = baos.toString(); verify(client).rollingUpdate(GROUP_NAME, JOB_ID, OPTIONS); assertEquals(1, ret); }
@Override protected int runWithJob(final Namespace options, final HeliosClient client, final PrintStream out, final boolean json, final Job job, final BufferedReader stdin) throws ExecutionException, InterruptedException, IOException { final JobId jobId = job.getId(); final String name = options.getString(nameArg.getDest()); final Long timeout = options.getLong(timeoutArg.getDest()); final Integer parallelism = options.getInt(parallelismArg.getDest()); final boolean async = options.getBoolean(asyncArg.getDest()); final long rolloutTimeout = options.getLong(rolloutTimeoutArg.getDest()); final Boolean migrate = options.getBoolean(migrateArg.getDest()); final Boolean overlap = options.getBoolean(overlapArg.getDest()); final String token = options.getString(tokenArg.getDest()); final Boolean ignoreFailures = options.getBoolean(ignoreFailuresArg.getDest()); checkArgument(timeout == null || timeout > 0, "Timeout must be greater than 0"); checkArgument(parallelism == null || parallelism > 0, "Parallelism must be greater than 0"); checkArgument(rolloutTimeout > 0, "Rollout timeout must be greater than 0"); final long startTime = timeSupplier.get(); final RolloutOptions rolloutOptions = RolloutOptions.newBuilder() .setTimeout(timeout) .setParallelism(parallelism) .setMigrate(migrate) .setOverlap(overlap) .setToken(token) .setIgnoreFailures(ignoreFailures) .build(); final RollingUpdateResponse response = client.rollingUpdate(name, jobId, rolloutOptions).get(); if (response.getStatus() != RollingUpdateResponse.Status.OK) { if (!json) { out.println("Failed: " + response); } else { out.println(response.toJsonString()); } return 1; } final RolloutOptions optionsFromJob = MoreObjects.firstNonNull(job.getRolloutOptions(), RolloutOptions.getDefault()); final Integer actualParallelism = nullableWithFallback(parallelism, optionsFromJob.getParallelism()); final Long actualTimeout = nullableWithFallback(timeout, optionsFromJob.getTimeout()); final Boolean actualOverlap = nullableWithFallback(overlap, optionsFromJob.getOverlap()); final String actualToken = nullableWithFallback(token, optionsFromJob.getToken()); final Boolean actualIgnoreFailures = nullableWithFallback(ignoreFailures, optionsFromJob.getIgnoreFailures()); final Boolean actualMigrate = nullableWithFallback(migrate, optionsFromJob.getMigrate()); if (!json) { out.println(format("Rolling update%s started: %s -> %s " + "(parallelism=%d, timeout=%d, overlap=%b, token=%s, " + "ignoreFailures=%b, migrate=%b)%s", async ? " (async)" : "", name, jobId.toShortString(), actualParallelism, actualTimeout, actualOverlap, actualToken, actualIgnoreFailures, actualMigrate, async ? "" : "\n")); } final Map<String, Object> jsonOutput = Maps.newHashMap(); jsonOutput.put("parallelism", actualParallelism); jsonOutput.put("timeout", actualTimeout); jsonOutput.put("overlap", actualOverlap); jsonOutput.put("token", actualToken); jsonOutput.put("ignoreFailures", actualIgnoreFailures); jsonOutput.put("migrate", actualMigrate); if (async) { if (json) { jsonOutput.put("status", response.getStatus()); out.println(Json.asStringUnchecked(jsonOutput)); } return 0; } String error = ""; boolean failed = false; boolean timedOut = false; final Set<String> reported = Sets.newHashSet(); while (true) { final DeploymentGroupStatusResponse status = client.deploymentGroupStatus(name).get(); if (status == null) { failed = true; error = "Failed to fetch deployment-group status"; break; } if (!jobId.equals(status.getDeploymentGroup().getJobId())) { failed = true; error = "Deployment-group job id changed during rolling-update"; break; } if (!json) { for (final DeploymentGroupStatusResponse.HostStatus hostStatus : status.getHostStatuses()) { final JobId hostJobId = hostStatus.getJobId(); final String host = hostStatus.getHost(); final TaskStatus.State state = hostStatus.getState(); final boolean done = hostJobId != null && hostJobId.equals(jobId) && state == TaskStatus.State.RUNNING; if (done && reported.add(host)) { out.println(format("%s -> %s (%d/%d)", host, state, reported.size(), status.getHostStatuses().size())); } } } if (status.getStatus() != DeploymentGroupStatusResponse.Status.ROLLING_OUT) { if (status.getStatus() == DeploymentGroupStatusResponse.Status.FAILED) { failed = true; error = status.getError(); } break; } if (timeSupplier.get() - startTime > TimeUnit.MINUTES.toMillis(rolloutTimeout)) { timedOut = true; break; } sleepFunction.sleep(POLL_INTERVAL_MILLIS); } final double duration = (timeSupplier.get() - startTime) / 1000.0; if (json) { if (failed) { jsonOutput.put("status", "FAILED"); jsonOutput.put("error", error); } else if (timedOut) { jsonOutput.put("status", "TIMEOUT"); } else { jsonOutput.put("status", "DONE"); } jsonOutput.put("duration", duration); out.println(Json.asStringUnchecked(jsonOutput)); } else { out.println(); if (failed) { out.println(format("Failed: %s", error)); } else if (timedOut) { out.println("Timed out! (rolling-update still in progress)"); } else { out.println("Done."); } out.println(format("Duration: %.2f s", duration)); } return (failed || timedOut) ? 1 : 0; }
RollingUpdateCommand extends WildcardJobCommand { @Override protected int runWithJob(final Namespace options, final HeliosClient client, final PrintStream out, final boolean json, final Job job, final BufferedReader stdin) throws ExecutionException, InterruptedException, IOException { final JobId jobId = job.getId(); final String name = options.getString(nameArg.getDest()); final Long timeout = options.getLong(timeoutArg.getDest()); final Integer parallelism = options.getInt(parallelismArg.getDest()); final boolean async = options.getBoolean(asyncArg.getDest()); final long rolloutTimeout = options.getLong(rolloutTimeoutArg.getDest()); final Boolean migrate = options.getBoolean(migrateArg.getDest()); final Boolean overlap = options.getBoolean(overlapArg.getDest()); final String token = options.getString(tokenArg.getDest()); final Boolean ignoreFailures = options.getBoolean(ignoreFailuresArg.getDest()); checkArgument(timeout == null || timeout > 0, "Timeout must be greater than 0"); checkArgument(parallelism == null || parallelism > 0, "Parallelism must be greater than 0"); checkArgument(rolloutTimeout > 0, "Rollout timeout must be greater than 0"); final long startTime = timeSupplier.get(); final RolloutOptions rolloutOptions = RolloutOptions.newBuilder() .setTimeout(timeout) .setParallelism(parallelism) .setMigrate(migrate) .setOverlap(overlap) .setToken(token) .setIgnoreFailures(ignoreFailures) .build(); final RollingUpdateResponse response = client.rollingUpdate(name, jobId, rolloutOptions).get(); if (response.getStatus() != RollingUpdateResponse.Status.OK) { if (!json) { out.println("Failed: " + response); } else { out.println(response.toJsonString()); } return 1; } final RolloutOptions optionsFromJob = MoreObjects.firstNonNull(job.getRolloutOptions(), RolloutOptions.getDefault()); final Integer actualParallelism = nullableWithFallback(parallelism, optionsFromJob.getParallelism()); final Long actualTimeout = nullableWithFallback(timeout, optionsFromJob.getTimeout()); final Boolean actualOverlap = nullableWithFallback(overlap, optionsFromJob.getOverlap()); final String actualToken = nullableWithFallback(token, optionsFromJob.getToken()); final Boolean actualIgnoreFailures = nullableWithFallback(ignoreFailures, optionsFromJob.getIgnoreFailures()); final Boolean actualMigrate = nullableWithFallback(migrate, optionsFromJob.getMigrate()); if (!json) { out.println(format("Rolling update%s started: %s -> %s " + "(parallelism=%d, timeout=%d, overlap=%b, token=%s, " + "ignoreFailures=%b, migrate=%b)%s", async ? " (async)" : "", name, jobId.toShortString(), actualParallelism, actualTimeout, actualOverlap, actualToken, actualIgnoreFailures, actualMigrate, async ? "" : "\n")); } final Map<String, Object> jsonOutput = Maps.newHashMap(); jsonOutput.put("parallelism", actualParallelism); jsonOutput.put("timeout", actualTimeout); jsonOutput.put("overlap", actualOverlap); jsonOutput.put("token", actualToken); jsonOutput.put("ignoreFailures", actualIgnoreFailures); jsonOutput.put("migrate", actualMigrate); if (async) { if (json) { jsonOutput.put("status", response.getStatus()); out.println(Json.asStringUnchecked(jsonOutput)); } return 0; } String error = ""; boolean failed = false; boolean timedOut = false; final Set<String> reported = Sets.newHashSet(); while (true) { final DeploymentGroupStatusResponse status = client.deploymentGroupStatus(name).get(); if (status == null) { failed = true; error = "Failed to fetch deployment-group status"; break; } if (!jobId.equals(status.getDeploymentGroup().getJobId())) { failed = true; error = "Deployment-group job id changed during rolling-update"; break; } if (!json) { for (final DeploymentGroupStatusResponse.HostStatus hostStatus : status.getHostStatuses()) { final JobId hostJobId = hostStatus.getJobId(); final String host = hostStatus.getHost(); final TaskStatus.State state = hostStatus.getState(); final boolean done = hostJobId != null && hostJobId.equals(jobId) && state == TaskStatus.State.RUNNING; if (done && reported.add(host)) { out.println(format("%s -> %s (%d/%d)", host, state, reported.size(), status.getHostStatuses().size())); } } } if (status.getStatus() != DeploymentGroupStatusResponse.Status.ROLLING_OUT) { if (status.getStatus() == DeploymentGroupStatusResponse.Status.FAILED) { failed = true; error = status.getError(); } break; } if (timeSupplier.get() - startTime > TimeUnit.MINUTES.toMillis(rolloutTimeout)) { timedOut = true; break; } sleepFunction.sleep(POLL_INTERVAL_MILLIS); } final double duration = (timeSupplier.get() - startTime) / 1000.0; if (json) { if (failed) { jsonOutput.put("status", "FAILED"); jsonOutput.put("error", error); } else if (timedOut) { jsonOutput.put("status", "TIMEOUT"); } else { jsonOutput.put("status", "DONE"); } jsonOutput.put("duration", duration); out.println(Json.asStringUnchecked(jsonOutput)); } else { out.println(); if (failed) { out.println(format("Failed: %s", error)); } else if (timedOut) { out.println("Timed out! (rolling-update still in progress)"); } else { out.println("Done."); } out.println(format("Duration: %.2f s", duration)); } return (failed || timedOut) ? 1 : 0; } }
RollingUpdateCommand extends WildcardJobCommand { @Override protected int runWithJob(final Namespace options, final HeliosClient client, final PrintStream out, final boolean json, final Job job, final BufferedReader stdin) throws ExecutionException, InterruptedException, IOException { final JobId jobId = job.getId(); final String name = options.getString(nameArg.getDest()); final Long timeout = options.getLong(timeoutArg.getDest()); final Integer parallelism = options.getInt(parallelismArg.getDest()); final boolean async = options.getBoolean(asyncArg.getDest()); final long rolloutTimeout = options.getLong(rolloutTimeoutArg.getDest()); final Boolean migrate = options.getBoolean(migrateArg.getDest()); final Boolean overlap = options.getBoolean(overlapArg.getDest()); final String token = options.getString(tokenArg.getDest()); final Boolean ignoreFailures = options.getBoolean(ignoreFailuresArg.getDest()); checkArgument(timeout == null || timeout > 0, "Timeout must be greater than 0"); checkArgument(parallelism == null || parallelism > 0, "Parallelism must be greater than 0"); checkArgument(rolloutTimeout > 0, "Rollout timeout must be greater than 0"); final long startTime = timeSupplier.get(); final RolloutOptions rolloutOptions = RolloutOptions.newBuilder() .setTimeout(timeout) .setParallelism(parallelism) .setMigrate(migrate) .setOverlap(overlap) .setToken(token) .setIgnoreFailures(ignoreFailures) .build(); final RollingUpdateResponse response = client.rollingUpdate(name, jobId, rolloutOptions).get(); if (response.getStatus() != RollingUpdateResponse.Status.OK) { if (!json) { out.println("Failed: " + response); } else { out.println(response.toJsonString()); } return 1; } final RolloutOptions optionsFromJob = MoreObjects.firstNonNull(job.getRolloutOptions(), RolloutOptions.getDefault()); final Integer actualParallelism = nullableWithFallback(parallelism, optionsFromJob.getParallelism()); final Long actualTimeout = nullableWithFallback(timeout, optionsFromJob.getTimeout()); final Boolean actualOverlap = nullableWithFallback(overlap, optionsFromJob.getOverlap()); final String actualToken = nullableWithFallback(token, optionsFromJob.getToken()); final Boolean actualIgnoreFailures = nullableWithFallback(ignoreFailures, optionsFromJob.getIgnoreFailures()); final Boolean actualMigrate = nullableWithFallback(migrate, optionsFromJob.getMigrate()); if (!json) { out.println(format("Rolling update%s started: %s -> %s " + "(parallelism=%d, timeout=%d, overlap=%b, token=%s, " + "ignoreFailures=%b, migrate=%b)%s", async ? " (async)" : "", name, jobId.toShortString(), actualParallelism, actualTimeout, actualOverlap, actualToken, actualIgnoreFailures, actualMigrate, async ? "" : "\n")); } final Map<String, Object> jsonOutput = Maps.newHashMap(); jsonOutput.put("parallelism", actualParallelism); jsonOutput.put("timeout", actualTimeout); jsonOutput.put("overlap", actualOverlap); jsonOutput.put("token", actualToken); jsonOutput.put("ignoreFailures", actualIgnoreFailures); jsonOutput.put("migrate", actualMigrate); if (async) { if (json) { jsonOutput.put("status", response.getStatus()); out.println(Json.asStringUnchecked(jsonOutput)); } return 0; } String error = ""; boolean failed = false; boolean timedOut = false; final Set<String> reported = Sets.newHashSet(); while (true) { final DeploymentGroupStatusResponse status = client.deploymentGroupStatus(name).get(); if (status == null) { failed = true; error = "Failed to fetch deployment-group status"; break; } if (!jobId.equals(status.getDeploymentGroup().getJobId())) { failed = true; error = "Deployment-group job id changed during rolling-update"; break; } if (!json) { for (final DeploymentGroupStatusResponse.HostStatus hostStatus : status.getHostStatuses()) { final JobId hostJobId = hostStatus.getJobId(); final String host = hostStatus.getHost(); final TaskStatus.State state = hostStatus.getState(); final boolean done = hostJobId != null && hostJobId.equals(jobId) && state == TaskStatus.State.RUNNING; if (done && reported.add(host)) { out.println(format("%s -> %s (%d/%d)", host, state, reported.size(), status.getHostStatuses().size())); } } } if (status.getStatus() != DeploymentGroupStatusResponse.Status.ROLLING_OUT) { if (status.getStatus() == DeploymentGroupStatusResponse.Status.FAILED) { failed = true; error = status.getError(); } break; } if (timeSupplier.get() - startTime > TimeUnit.MINUTES.toMillis(rolloutTimeout)) { timedOut = true; break; } sleepFunction.sleep(POLL_INTERVAL_MILLIS); } final double duration = (timeSupplier.get() - startTime) / 1000.0; if (json) { if (failed) { jsonOutput.put("status", "FAILED"); jsonOutput.put("error", error); } else if (timedOut) { jsonOutput.put("status", "TIMEOUT"); } else { jsonOutput.put("status", "DONE"); } jsonOutput.put("duration", duration); out.println(Json.asStringUnchecked(jsonOutput)); } else { out.println(); if (failed) { out.println(format("Failed: %s", error)); } else if (timedOut) { out.println("Timed out! (rolling-update still in progress)"); } else { out.println("Done."); } out.println(format("Duration: %.2f s", duration)); } return (failed || timedOut) ? 1 : 0; } RollingUpdateCommand(final Subparser parser); @VisibleForTesting  RollingUpdateCommand(final Subparser parser, final SleepFunction sleepFunction,
                       final Supplier<Long> timeSupplier); }
RollingUpdateCommand extends WildcardJobCommand { @Override protected int runWithJob(final Namespace options, final HeliosClient client, final PrintStream out, final boolean json, final Job job, final BufferedReader stdin) throws ExecutionException, InterruptedException, IOException { final JobId jobId = job.getId(); final String name = options.getString(nameArg.getDest()); final Long timeout = options.getLong(timeoutArg.getDest()); final Integer parallelism = options.getInt(parallelismArg.getDest()); final boolean async = options.getBoolean(asyncArg.getDest()); final long rolloutTimeout = options.getLong(rolloutTimeoutArg.getDest()); final Boolean migrate = options.getBoolean(migrateArg.getDest()); final Boolean overlap = options.getBoolean(overlapArg.getDest()); final String token = options.getString(tokenArg.getDest()); final Boolean ignoreFailures = options.getBoolean(ignoreFailuresArg.getDest()); checkArgument(timeout == null || timeout > 0, "Timeout must be greater than 0"); checkArgument(parallelism == null || parallelism > 0, "Parallelism must be greater than 0"); checkArgument(rolloutTimeout > 0, "Rollout timeout must be greater than 0"); final long startTime = timeSupplier.get(); final RolloutOptions rolloutOptions = RolloutOptions.newBuilder() .setTimeout(timeout) .setParallelism(parallelism) .setMigrate(migrate) .setOverlap(overlap) .setToken(token) .setIgnoreFailures(ignoreFailures) .build(); final RollingUpdateResponse response = client.rollingUpdate(name, jobId, rolloutOptions).get(); if (response.getStatus() != RollingUpdateResponse.Status.OK) { if (!json) { out.println("Failed: " + response); } else { out.println(response.toJsonString()); } return 1; } final RolloutOptions optionsFromJob = MoreObjects.firstNonNull(job.getRolloutOptions(), RolloutOptions.getDefault()); final Integer actualParallelism = nullableWithFallback(parallelism, optionsFromJob.getParallelism()); final Long actualTimeout = nullableWithFallback(timeout, optionsFromJob.getTimeout()); final Boolean actualOverlap = nullableWithFallback(overlap, optionsFromJob.getOverlap()); final String actualToken = nullableWithFallback(token, optionsFromJob.getToken()); final Boolean actualIgnoreFailures = nullableWithFallback(ignoreFailures, optionsFromJob.getIgnoreFailures()); final Boolean actualMigrate = nullableWithFallback(migrate, optionsFromJob.getMigrate()); if (!json) { out.println(format("Rolling update%s started: %s -> %s " + "(parallelism=%d, timeout=%d, overlap=%b, token=%s, " + "ignoreFailures=%b, migrate=%b)%s", async ? " (async)" : "", name, jobId.toShortString(), actualParallelism, actualTimeout, actualOverlap, actualToken, actualIgnoreFailures, actualMigrate, async ? "" : "\n")); } final Map<String, Object> jsonOutput = Maps.newHashMap(); jsonOutput.put("parallelism", actualParallelism); jsonOutput.put("timeout", actualTimeout); jsonOutput.put("overlap", actualOverlap); jsonOutput.put("token", actualToken); jsonOutput.put("ignoreFailures", actualIgnoreFailures); jsonOutput.put("migrate", actualMigrate); if (async) { if (json) { jsonOutput.put("status", response.getStatus()); out.println(Json.asStringUnchecked(jsonOutput)); } return 0; } String error = ""; boolean failed = false; boolean timedOut = false; final Set<String> reported = Sets.newHashSet(); while (true) { final DeploymentGroupStatusResponse status = client.deploymentGroupStatus(name).get(); if (status == null) { failed = true; error = "Failed to fetch deployment-group status"; break; } if (!jobId.equals(status.getDeploymentGroup().getJobId())) { failed = true; error = "Deployment-group job id changed during rolling-update"; break; } if (!json) { for (final DeploymentGroupStatusResponse.HostStatus hostStatus : status.getHostStatuses()) { final JobId hostJobId = hostStatus.getJobId(); final String host = hostStatus.getHost(); final TaskStatus.State state = hostStatus.getState(); final boolean done = hostJobId != null && hostJobId.equals(jobId) && state == TaskStatus.State.RUNNING; if (done && reported.add(host)) { out.println(format("%s -> %s (%d/%d)", host, state, reported.size(), status.getHostStatuses().size())); } } } if (status.getStatus() != DeploymentGroupStatusResponse.Status.ROLLING_OUT) { if (status.getStatus() == DeploymentGroupStatusResponse.Status.FAILED) { failed = true; error = status.getError(); } break; } if (timeSupplier.get() - startTime > TimeUnit.MINUTES.toMillis(rolloutTimeout)) { timedOut = true; break; } sleepFunction.sleep(POLL_INTERVAL_MILLIS); } final double duration = (timeSupplier.get() - startTime) / 1000.0; if (json) { if (failed) { jsonOutput.put("status", "FAILED"); jsonOutput.put("error", error); } else if (timedOut) { jsonOutput.put("status", "TIMEOUT"); } else { jsonOutput.put("status", "DONE"); } jsonOutput.put("duration", duration); out.println(Json.asStringUnchecked(jsonOutput)); } else { out.println(); if (failed) { out.println(format("Failed: %s", error)); } else if (timedOut) { out.println("Timed out! (rolling-update still in progress)"); } else { out.println("Done."); } out.println(format("Duration: %.2f s", duration)); } return (failed || timedOut) ? 1 : 0; } RollingUpdateCommand(final Subparser parser); @VisibleForTesting  RollingUpdateCommand(final Subparser parser, final SleepFunction sleepFunction,
                       final Supplier<Long> timeSupplier);  }
RollingUpdateCommand extends WildcardJobCommand { @Override protected int runWithJob(final Namespace options, final HeliosClient client, final PrintStream out, final boolean json, final Job job, final BufferedReader stdin) throws ExecutionException, InterruptedException, IOException { final JobId jobId = job.getId(); final String name = options.getString(nameArg.getDest()); final Long timeout = options.getLong(timeoutArg.getDest()); final Integer parallelism = options.getInt(parallelismArg.getDest()); final boolean async = options.getBoolean(asyncArg.getDest()); final long rolloutTimeout = options.getLong(rolloutTimeoutArg.getDest()); final Boolean migrate = options.getBoolean(migrateArg.getDest()); final Boolean overlap = options.getBoolean(overlapArg.getDest()); final String token = options.getString(tokenArg.getDest()); final Boolean ignoreFailures = options.getBoolean(ignoreFailuresArg.getDest()); checkArgument(timeout == null || timeout > 0, "Timeout must be greater than 0"); checkArgument(parallelism == null || parallelism > 0, "Parallelism must be greater than 0"); checkArgument(rolloutTimeout > 0, "Rollout timeout must be greater than 0"); final long startTime = timeSupplier.get(); final RolloutOptions rolloutOptions = RolloutOptions.newBuilder() .setTimeout(timeout) .setParallelism(parallelism) .setMigrate(migrate) .setOverlap(overlap) .setToken(token) .setIgnoreFailures(ignoreFailures) .build(); final RollingUpdateResponse response = client.rollingUpdate(name, jobId, rolloutOptions).get(); if (response.getStatus() != RollingUpdateResponse.Status.OK) { if (!json) { out.println("Failed: " + response); } else { out.println(response.toJsonString()); } return 1; } final RolloutOptions optionsFromJob = MoreObjects.firstNonNull(job.getRolloutOptions(), RolloutOptions.getDefault()); final Integer actualParallelism = nullableWithFallback(parallelism, optionsFromJob.getParallelism()); final Long actualTimeout = nullableWithFallback(timeout, optionsFromJob.getTimeout()); final Boolean actualOverlap = nullableWithFallback(overlap, optionsFromJob.getOverlap()); final String actualToken = nullableWithFallback(token, optionsFromJob.getToken()); final Boolean actualIgnoreFailures = nullableWithFallback(ignoreFailures, optionsFromJob.getIgnoreFailures()); final Boolean actualMigrate = nullableWithFallback(migrate, optionsFromJob.getMigrate()); if (!json) { out.println(format("Rolling update%s started: %s -> %s " + "(parallelism=%d, timeout=%d, overlap=%b, token=%s, " + "ignoreFailures=%b, migrate=%b)%s", async ? " (async)" : "", name, jobId.toShortString(), actualParallelism, actualTimeout, actualOverlap, actualToken, actualIgnoreFailures, actualMigrate, async ? "" : "\n")); } final Map<String, Object> jsonOutput = Maps.newHashMap(); jsonOutput.put("parallelism", actualParallelism); jsonOutput.put("timeout", actualTimeout); jsonOutput.put("overlap", actualOverlap); jsonOutput.put("token", actualToken); jsonOutput.put("ignoreFailures", actualIgnoreFailures); jsonOutput.put("migrate", actualMigrate); if (async) { if (json) { jsonOutput.put("status", response.getStatus()); out.println(Json.asStringUnchecked(jsonOutput)); } return 0; } String error = ""; boolean failed = false; boolean timedOut = false; final Set<String> reported = Sets.newHashSet(); while (true) { final DeploymentGroupStatusResponse status = client.deploymentGroupStatus(name).get(); if (status == null) { failed = true; error = "Failed to fetch deployment-group status"; break; } if (!jobId.equals(status.getDeploymentGroup().getJobId())) { failed = true; error = "Deployment-group job id changed during rolling-update"; break; } if (!json) { for (final DeploymentGroupStatusResponse.HostStatus hostStatus : status.getHostStatuses()) { final JobId hostJobId = hostStatus.getJobId(); final String host = hostStatus.getHost(); final TaskStatus.State state = hostStatus.getState(); final boolean done = hostJobId != null && hostJobId.equals(jobId) && state == TaskStatus.State.RUNNING; if (done && reported.add(host)) { out.println(format("%s -> %s (%d/%d)", host, state, reported.size(), status.getHostStatuses().size())); } } } if (status.getStatus() != DeploymentGroupStatusResponse.Status.ROLLING_OUT) { if (status.getStatus() == DeploymentGroupStatusResponse.Status.FAILED) { failed = true; error = status.getError(); } break; } if (timeSupplier.get() - startTime > TimeUnit.MINUTES.toMillis(rolloutTimeout)) { timedOut = true; break; } sleepFunction.sleep(POLL_INTERVAL_MILLIS); } final double duration = (timeSupplier.get() - startTime) / 1000.0; if (json) { if (failed) { jsonOutput.put("status", "FAILED"); jsonOutput.put("error", error); } else if (timedOut) { jsonOutput.put("status", "TIMEOUT"); } else { jsonOutput.put("status", "DONE"); } jsonOutput.put("duration", duration); out.println(Json.asStringUnchecked(jsonOutput)); } else { out.println(); if (failed) { out.println(format("Failed: %s", error)); } else if (timedOut) { out.println("Timed out! (rolling-update still in progress)"); } else { out.println("Done."); } out.println(format("Duration: %.2f s", duration)); } return (failed || timedOut) ? 1 : 0; } RollingUpdateCommand(final Subparser parser); @VisibleForTesting  RollingUpdateCommand(final Subparser parser, final SleepFunction sleepFunction,
                       final Supplier<Long> timeSupplier);   }