KubeMonitorJob implements Job { String check(KubePodList.Pod pod, ZonedDateTime now) { if (pod.metadata.deletionTimestamp != null) { Duration elapsed = Duration.between(pod.metadata.deletionTimestamp, now); if (elapsed.toSeconds() >= 300) { return "pod is still in deletion, elapsed=" + elapsed; } return null; } String phase = pod.status.phase; if ("Succeeded".equals(phase)) return null; if ("Failed".equals(phase) || "Unknown".equals(phase)) return "unexpected pod phase, phase=" + phase; if ("Pending".equals(phase)) { for (KubePodList.ContainerStatus status : pod.status.containerStatuses) { if (status.state.waiting != null && "ImagePullBackOff".equals(status.state.waiting.reason)) { return "ImagePullBackOff: " + status.state.waiting.message; } } } if ("Running".equals(phase)) { boolean ready = true; for (KubePodList.ContainerStatus status : pod.status.containerStatuses) { if (status.state.waiting != null && "CrashLoopBackOff".equals(status.state.waiting.reason)) { return "CrashLoopBackOff: " + status.state.waiting.message; } boolean containerReady = Boolean.TRUE.equals(status.ready); if (!containerReady && status.restartCount >= 5) { return "pod restarted too many times, restart=" + status.restartCount; } if (!containerReady) { ready = false; } } if (ready) return null; } ZonedDateTime startTime = pod.status.startTime != null ? pod.status.startTime : pod.metadata.creationTimestamp; Duration elapsed = Duration.between(startTime, now); if (elapsed.toSeconds() >= 300) { return "pod is not in ready state, uptime=" + elapsed; } return null; } KubeMonitorJob(List<String> namespaces, KubeClient kubeClient, MessagePublisher<StatMessage> publisher); @Override void execute(JobContext context); final MessagePublisher<StatMessage> publisher; final KubeClient kubeClient; final List<String> namespaces; }
@Test void checkWithTermination() { var deleteTime = ZonedDateTime.now(); var pod = pod("Pending"); pod.metadata.deletionTimestamp = deleteTime; assertThat(job.check(pod, deleteTime.plusMinutes(1))).isNull(); assertThat(job.check(pod, deleteTime.plusMinutes(5))).startsWith("pod is still in deletion, elapsed="); }
@Test void checkWithCrashLoopBackOff() { var pod = pod("Running"); KubePodList.ContainerStatus status = pod.status.containerStatuses.get(0); status.state.waiting = new KubePodList.ContainerStateWaiting(); status.state.waiting.reason = "CrashLoopBackOff"; status.state.waiting.message = "Back-off 5m0s restarting failed container"; assertThat(job.check(pod, ZonedDateTime.now())) .contains(status.state.waiting.reason) .contains(status.state.waiting.message); }
@Test void checkWithNotReady() { var startTime = ZonedDateTime.now(); var pod = pod("Pending"); pod.metadata.creationTimestamp = startTime; assertThat(job.check(pod, startTime.plusMinutes(1))).isNull(); assertThat(job.check(pod, startTime.plusMinutes(5))).isEqualTo("pod is not in ready state, uptime=PT5M"); pod = pod("Running"); pod.status.containerStatuses.get(0).ready = Boolean.FALSE; pod.status.startTime = startTime; assertThat(job.check(pod, startTime.plusMinutes(1))).isNull(); assertThat(job.check(pod, startTime.plusMinutes(5))).isEqualTo("pod is not in ready state, uptime=PT5M"); }
@Test void checkWithTooManyRestarts() { var pod = pod("Running"); KubePodList.ContainerStatus status = pod.status.containerStatuses.get(0); status.ready = Boolean.FALSE; status.restartCount = 5; assertThat(job.check(pod, ZonedDateTime.now())).isEqualTo("pod restarted too many times, restart=5"); status.ready = Boolean.TRUE; assertThat(job.check(pod, ZonedDateTime.now())).isNull(); }
@Test void checkWithImagePullBackOff() { var pod = pod("Pending"); KubePodList.ContainerStatus status = pod.status.containerStatuses.get(0); status.state.waiting = new KubePodList.ContainerStateWaiting(); status.state.waiting.reason = "ImagePullBackOff"; status.state.waiting.message = "Back-off pulling image \"gcr.io/project/ops/debug:latest\""; assertThat(job.check(pod, ZonedDateTime.now())) .contains(status.state.waiting.reason) .contains(status.state.waiting.message); }