@Test public void testAutoRemediationRateLimit() throws Exception { final InputStream mockOutput = new ByteArrayInputStream("".getBytes()); instanceState.setShouldCassandraBeAlive(true); instanceState.markLastAttemptedStartTime(); new Expectations() { { mockProcess.getInputStream(); result = mockOutput; times = 12; cassProcess.start(true); times = 2; } }; final Runtime r = Runtime.getRuntime(); String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() }; new Expectations(r) { { r.exec(cmd); result = mockProcess; } }; monitor.execute(); Thread.sleep(1500); monitor.execute(); monitor.execute(); Thread.sleep(1500); monitor.execute(); monitor.execute(); monitor.execute(); new Verifications() {}; }
@Override public void execute() throws Exception { try { checkRequiredDirectories(); instanceState.setIsRequiredDirectoriesExist(true); } catch (IllegalStateException e) { instanceState.setIsRequiredDirectoriesExist(false); } Process process = null; BufferedReader input = null; try { final String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() }; process = Runtime.getRuntime().exec(cmd); input = new BufferedReader(new InputStreamReader(process.getInputStream())); String line = input.readLine(); if (line != null) { instanceState.setCassandraProcessAlive(true); isCassandraStarted.set(true); NodeProbe bean = JMXNodeTool.instance(this.config); instanceState.setIsGossipActive(bean.isGossipRunning()); instanceState.setIsNativeTransportActive(bean.isNativeTransportRunning()); instanceState.setIsThriftActive(bean.isThriftServerRunning()); } else { instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } } catch (Exception e) { logger.warn("Exception thrown while checking if Cassandra is running or not ", e); instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } finally { if (process != null) { IOUtils.closeQuietly(process.getInputStream()); IOUtils.closeQuietly(process.getOutputStream()); IOUtils.closeQuietly(process.getErrorStream()); } if (input != null) IOUtils.closeQuietly(input); } try { int rate = config.getRemediateDeadCassandraRate(); if (rate >= 0 && !config.doesCassandraStartManually()) { if (instanceState.shouldCassandraBeAlive() && !instanceState.isCassandraProcessAlive()) { long msNow = System.currentTimeMillis(); if (rate == 0 || ((instanceState.getLastAttemptedStartTime() + rate * 1000) < msNow)) { cassMonitorMetrics.incCassAutoStart(); cassProcess.start(true); instanceState.markLastAttemptedStartTime(); } } } } catch (IOException e) { logger.warn("Failed to remediate dead Cassandra", e); } }
CassandraMonitor extends Task { @Override public void execute() throws Exception { try { checkRequiredDirectories(); instanceState.setIsRequiredDirectoriesExist(true); } catch (IllegalStateException e) { instanceState.setIsRequiredDirectoriesExist(false); } Process process = null; BufferedReader input = null; try { final String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() }; process = Runtime.getRuntime().exec(cmd); input = new BufferedReader(new InputStreamReader(process.getInputStream())); String line = input.readLine(); if (line != null) { instanceState.setCassandraProcessAlive(true); isCassandraStarted.set(true); NodeProbe bean = JMXNodeTool.instance(this.config); instanceState.setIsGossipActive(bean.isGossipRunning()); instanceState.setIsNativeTransportActive(bean.isNativeTransportRunning()); instanceState.setIsThriftActive(bean.isThriftServerRunning()); } else { instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } } catch (Exception e) { logger.warn("Exception thrown while checking if Cassandra is running or not ", e); instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } finally { if (process != null) { IOUtils.closeQuietly(process.getInputStream()); IOUtils.closeQuietly(process.getOutputStream()); IOUtils.closeQuietly(process.getErrorStream()); } if (input != null) IOUtils.closeQuietly(input); } try { int rate = config.getRemediateDeadCassandraRate(); if (rate >= 0 && !config.doesCassandraStartManually()) { if (instanceState.shouldCassandraBeAlive() && !instanceState.isCassandraProcessAlive()) { long msNow = System.currentTimeMillis(); if (rate == 0 || ((instanceState.getLastAttemptedStartTime() + rate * 1000) < msNow)) { cassMonitorMetrics.incCassAutoStart(); cassProcess.start(true); instanceState.markLastAttemptedStartTime(); } } } } catch (IOException e) { logger.warn("Failed to remediate dead Cassandra", e); } } }
CassandraMonitor extends Task { @Override public void execute() throws Exception { try { checkRequiredDirectories(); instanceState.setIsRequiredDirectoriesExist(true); } catch (IllegalStateException e) { instanceState.setIsRequiredDirectoriesExist(false); } Process process = null; BufferedReader input = null; try { final String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() }; process = Runtime.getRuntime().exec(cmd); input = new BufferedReader(new InputStreamReader(process.getInputStream())); String line = input.readLine(); if (line != null) { instanceState.setCassandraProcessAlive(true); isCassandraStarted.set(true); NodeProbe bean = JMXNodeTool.instance(this.config); instanceState.setIsGossipActive(bean.isGossipRunning()); instanceState.setIsNativeTransportActive(bean.isNativeTransportRunning()); instanceState.setIsThriftActive(bean.isThriftServerRunning()); } else { instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } } catch (Exception e) { logger.warn("Exception thrown while checking if Cassandra is running or not ", e); instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } finally { if (process != null) { IOUtils.closeQuietly(process.getInputStream()); IOUtils.closeQuietly(process.getOutputStream()); IOUtils.closeQuietly(process.getErrorStream()); } if (input != null) IOUtils.closeQuietly(input); } try { int rate = config.getRemediateDeadCassandraRate(); if (rate >= 0 && !config.doesCassandraStartManually()) { if (instanceState.shouldCassandraBeAlive() && !instanceState.isCassandraProcessAlive()) { long msNow = System.currentTimeMillis(); if (rate == 0 || ((instanceState.getLastAttemptedStartTime() + rate * 1000) < msNow)) { cassMonitorMetrics.incCassAutoStart(); cassProcess.start(true); instanceState.markLastAttemptedStartTime(); } } } } catch (IOException e) { logger.warn("Failed to remediate dead Cassandra", e); } } @Inject protected  CassandraMonitor(
            IConfiguration config,
            InstanceState instanceState,
            ICassandraProcess cassProcess,
            CassMonitorMetrics cassMonitorMetrics); }
CassandraMonitor extends Task { @Override public void execute() throws Exception { try { checkRequiredDirectories(); instanceState.setIsRequiredDirectoriesExist(true); } catch (IllegalStateException e) { instanceState.setIsRequiredDirectoriesExist(false); } Process process = null; BufferedReader input = null; try { final String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() }; process = Runtime.getRuntime().exec(cmd); input = new BufferedReader(new InputStreamReader(process.getInputStream())); String line = input.readLine(); if (line != null) { instanceState.setCassandraProcessAlive(true); isCassandraStarted.set(true); NodeProbe bean = JMXNodeTool.instance(this.config); instanceState.setIsGossipActive(bean.isGossipRunning()); instanceState.setIsNativeTransportActive(bean.isNativeTransportRunning()); instanceState.setIsThriftActive(bean.isThriftServerRunning()); } else { instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } } catch (Exception e) { logger.warn("Exception thrown while checking if Cassandra is running or not ", e); instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } finally { if (process != null) { IOUtils.closeQuietly(process.getInputStream()); IOUtils.closeQuietly(process.getOutputStream()); IOUtils.closeQuietly(process.getErrorStream()); } if (input != null) IOUtils.closeQuietly(input); } try { int rate = config.getRemediateDeadCassandraRate(); if (rate >= 0 && !config.doesCassandraStartManually()) { if (instanceState.shouldCassandraBeAlive() && !instanceState.isCassandraProcessAlive()) { long msNow = System.currentTimeMillis(); if (rate == 0 || ((instanceState.getLastAttemptedStartTime() + rate * 1000) < msNow)) { cassMonitorMetrics.incCassAutoStart(); cassProcess.start(true); instanceState.markLastAttemptedStartTime(); } } } } catch (IOException e) { logger.warn("Failed to remediate dead Cassandra", e); } } @Inject protected  CassandraMonitor(
            IConfiguration config,
            InstanceState instanceState,
            ICassandraProcess cassProcess,
            CassMonitorMetrics cassMonitorMetrics); @Override void execute(); static TaskTimer getTimer(); @Override String getName(); static Boolean hasCassadraStarted(); static void setIsCassadraStarted(); }
CassandraMonitor extends Task { @Override public void execute() throws Exception { try { checkRequiredDirectories(); instanceState.setIsRequiredDirectoriesExist(true); } catch (IllegalStateException e) { instanceState.setIsRequiredDirectoriesExist(false); } Process process = null; BufferedReader input = null; try { final String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() }; process = Runtime.getRuntime().exec(cmd); input = new BufferedReader(new InputStreamReader(process.getInputStream())); String line = input.readLine(); if (line != null) { instanceState.setCassandraProcessAlive(true); isCassandraStarted.set(true); NodeProbe bean = JMXNodeTool.instance(this.config); instanceState.setIsGossipActive(bean.isGossipRunning()); instanceState.setIsNativeTransportActive(bean.isNativeTransportRunning()); instanceState.setIsThriftActive(bean.isThriftServerRunning()); } else { instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } } catch (Exception e) { logger.warn("Exception thrown while checking if Cassandra is running or not ", e); instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } finally { if (process != null) { IOUtils.closeQuietly(process.getInputStream()); IOUtils.closeQuietly(process.getOutputStream()); IOUtils.closeQuietly(process.getErrorStream()); } if (input != null) IOUtils.closeQuietly(input); } try { int rate = config.getRemediateDeadCassandraRate(); if (rate >= 0 && !config.doesCassandraStartManually()) { if (instanceState.shouldCassandraBeAlive() && !instanceState.isCassandraProcessAlive()) { long msNow = System.currentTimeMillis(); if (rate == 0 || ((instanceState.getLastAttemptedStartTime() + rate * 1000) < msNow)) { cassMonitorMetrics.incCassAutoStart(); cassProcess.start(true); instanceState.markLastAttemptedStartTime(); } } } } catch (IOException e) { logger.warn("Failed to remediate dead Cassandra", e); } } @Inject protected  CassandraMonitor(
            IConfiguration config,
            InstanceState instanceState,
            ICassandraProcess cassProcess,
            CassMonitorMetrics cassMonitorMetrics); @Override void execute(); static TaskTimer getTimer(); @Override String getName(); static Boolean hasCassadraStarted(); static void setIsCassadraStarted(); static final String JOBNAME; }