InconsistentRuntimesDetector { @VisibleForTesting void checkOne(String workspaceId) throws InfrastructureException { LOG.debug("Checking consistency of runtime for workspace `{}`", workspaceId); KubernetesInternalRuntime k8sRuntime = getKubernetesInternalRuntime(workspaceId); RuntimeIdentity runtimeId = k8sRuntime.getContext().getIdentity(); try { if (k8sRuntime.isConsistent()) { return; } } catch (InfrastructureException e) { throw new InfrastructureException( format( "Error occurred during runtime '%s:%s' consistency checking. Cause: %s", runtimeId.getWorkspaceId(), runtimeId.getOwnerId(), e.getMessage()), e); } if (!isRunning(k8sRuntime)) { return; } LOG.warn( "Found runtime `{}:{}` with inconsistent state. It's going to be stopped automatically", runtimeId.getWorkspaceId(), runtimeId.getOwnerId()); stopAbnormally(k8sRuntime); LOG.debug("Checking consistency of runtime for workspace `{}` is finished", workspaceId); } @Inject  InconsistentRuntimesDetector(
      RuntimeEventsPublisher eventPublisher, WorkspaceRuntimes workspaceRuntimes); @ScheduleDelay( delayParameterName = "che.infra.kubernetes.runtimes_consistency_check_period_min", initialDelayParameterName = "che.infra.kubernetes.runtimes_consistency_check_period_min", unit = TimeUnit.MINUTES) void check();  }
@Test public void shouldDoNothingIfRuntimeHasConsistentStateOnChecking() throws Exception { doReturn(k8sRuntime).when(workspaceRuntimes).getInternalRuntime("workspace1"); doReturn(true).when(k8sRuntime).isConsistent(); inconsistentRuntimesDetector.checkOne("workspace1"); verifyNoMoreInteractions(eventPublisher); }
@Test( expectedExceptions = InfrastructureException.class, expectedExceptionsMessageRegExp = "Fetched internal runtime 'workspace1:owner1' is not Kubernetes, it is not possible to check consistency.") public void shouldThrowExceptionIfNonK8sRuntimeIsReturnOnCheckingConsistency() throws Exception { InternalRuntime runtime = mock(InternalRuntime.class); when(runtime.getContext()).thenReturn(k8sContext); doReturn(runtime).when(workspaceRuntimes).getInternalRuntime("workspace1"); inconsistentRuntimesDetector.checkOne("workspace1"); verifyNoMoreInteractions(eventPublisher); }
@Test( expectedExceptions = InfrastructureException.class, expectedExceptionsMessageRegExp = "Failed to get internal runtime for workspace `workspace1` to check consistency. Cause: error") public void shouldThrowExceptionIfExceptionOccursDuringRuntimeFetchingOnCheckingConsistency() throws Exception { doThrow(new InfrastructureException("error")) .when(workspaceRuntimes) .getInternalRuntime("workspace1"); inconsistentRuntimesDetector.checkOne("workspace1"); verifyNoMoreInteractions(eventPublisher); }
@Test( expectedExceptions = InfrastructureException.class, expectedExceptionsMessageRegExp = "Error occurred during runtime 'workspace1:owner1' consistency checking. Cause: error") public void shouldThrowExceptionIfExceptionOccursOnCheckingConsistency() throws Exception { doReturn(k8sRuntime).when(workspaceRuntimes).getInternalRuntime("workspace1"); doThrow(new InfrastructureException("error")).when(k8sRuntime).isConsistent(); inconsistentRuntimesDetector.checkOne("workspace1"); verifyNoMoreInteractions(eventPublisher); }
@Test public void shouldMarkRuntimeAsStoppedEvenWhenExceptionOccursDuringRuntimeStoppingOnCheckingConsistency() throws Exception { doReturn(k8sRuntime).when(workspaceRuntimes).getInternalRuntime("workspace1"); doReturn(false).when(k8sRuntime).isConsistent(); doThrow(new InfrastructureException("error")).when(k8sRuntime).stop(any()); InfrastructureException caughtException = null; try { inconsistentRuntimesDetector.checkOne("workspace1"); } catch (InfrastructureException e) { caughtException = e; } assertNotNull(caughtException); assertEquals( caughtException.getMessage(), "Failed to stop the runtime 'workspace1:owner1' which has inconsistent state. Error: error"); verify(k8sRuntime).stop(emptyMap()); verify(eventPublisher) .sendAbnormalStoppingEvent(runtimeId, "The runtime has inconsistent state."); verify(eventPublisher) .sendAbnormalStoppedEvent(runtimeId, "The runtime has inconsistent state."); }
@Test public void shouldStopRuntimeAbnormallyIfRuntimeHasInConsistentStateOnChecking() throws Exception { doReturn(k8sRuntime).when(workspaceRuntimes).getInternalRuntime("workspace1"); doReturn(false).when(k8sRuntime).isConsistent(); inconsistentRuntimesDetector.checkOne("workspace1"); verify(k8sRuntime).stop(emptyMap()); verify(eventPublisher) .sendAbnormalStoppingEvent(runtimeId, "The runtime has inconsistent state."); verify(eventPublisher) .sendAbnormalStoppedEvent(runtimeId, "The runtime has inconsistent state."); }
@Test public void shouldNotStopRuntimeAbnormallyIfRuntimeHasInconsistentStateButIsNotRunningAnyMore() throws Exception { doReturn(k8sRuntime).when(workspaceRuntimes).getInternalRuntime("workspace1"); doReturn(false).when(k8sRuntime).isConsistent(); when(k8sRuntime.getStatus()).thenReturn(WorkspaceStatus.STOPPING); inconsistentRuntimesDetector.checkOne("workspace1"); verify(k8sRuntime, never()).stop(emptyMap()); verify(eventPublisher, never()) .sendAbnormalStoppingEvent(runtimeId, "The runtime has inconsistent state."); verify(eventPublisher, never()) .sendAbnormalStoppedEvent(runtimeId, "The runtime has inconsistent state."); }