@@ -59,6 +59,7 @@ public class JdbcExecutorLoaderTest {
   private static final String user = "azkaban";
   private static final String password = "azkaban";
   private static final int numConnections = 10;
+  private static final long recentlyFinishedLifetimeMs = 5000;
 
   @BeforeClass
   public static void setupDB() {
/*
 * Copyright 2014 LinkedIn Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package azkaban.executor;

import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import javax.sql.DataSource;

import org.apache.commons.dbutils.DbUtils;
import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.ResultSetHandler;
import org.joda.time.DateTime;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;

import azkaban.database.DataSourceUtils;
import azkaban.executor.ExecutorLogEvent.EventType;
import azkaban.user.User;
import azkaban.utils.FileIOUtils.LogData;
import azkaban.utils.Pair;
import azkaban.utils.Props;
import azkaban.utils.TestUtils;

public class JdbcExecutorLoaderTest {
  private static boolean testDBExists;
  /* Directory with serialized description of test flows */
  private static final String UNIT_BASE_DIR =
    "../azkaban-test/src/test/resources/azkaban/test/executions";
  // @TODO remove this and turn into local host.
  private static final String host = "localhost";
  private static final int port = 3306;
  private static final String database = "azkaban2";
  private static final String user = "azkaban";
  private static final String password = "azkaban";
  private static final int numConnections = 10;

  @BeforeClass
  public static void setupDB() {
    DataSource dataSource =
        DataSourceUtils.getMySQLDataSource(host, port, database, user,
            password, numConnections);
    testDBExists = true;

    Connection connection = null;
    try {
      connection = dataSource.getConnection();
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    CountHandler countHandler = new CountHandler();
    QueryRunner runner = new QueryRunner();
    try {
      runner.query(connection, "SELECT COUNT(1) FROM active_executing_flows",
          countHandler);
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.query(connection, "SELECT COUNT(1) FROM execution_flows",
          countHandler);
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.query(connection, "SELECT COUNT(1) FROM execution_jobs",
          countHandler);
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.query(connection, "SELECT COUNT(1) FROM execution_logs",
          countHandler);
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.query(connection, "SELECT COUNT(1) FROM executors",
          countHandler);
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.query(connection, "SELECT COUNT(1) FROM executor_events",
          countHandler);
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    DbUtils.closeQuietly(connection);
  }

  @After
  public void clearDB() {
    if (!testDBExists) {
      return;
    }

    DataSource dataSource =
        DataSourceUtils.getMySQLDataSource(host, port, database, user,
            password, numConnections);
    Connection connection = null;
    try {
      connection = dataSource.getConnection();
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    QueryRunner runner = new QueryRunner();
    try {
      runner.update(connection, "DELETE FROM active_executing_flows");

    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.update(connection, "DELETE FROM execution_flows");
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.update(connection, "DELETE FROM execution_jobs");
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.update(connection, "DELETE FROM execution_logs");
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.update(connection, "DELETE FROM executors");
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    try {
      runner.update(connection, "DELETE FROM executor_events");
    } catch (SQLException e) {
      e.printStackTrace();
      testDBExists = false;
      DbUtils.closeQuietly(connection);
      return;
    }

    DbUtils.closeQuietly(connection);
  }

  @Test
  public void testUploadExecutionFlows() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    ExecutableFlow flow = TestUtils.createExecutableFlow("exectest1", "exec1");

    loader.uploadExecutableFlow(flow);

    ExecutableFlow fetchFlow =
        loader.fetchExecutableFlow(flow.getExecutionId());

    // Shouldn't be the same object.
    Assert.assertTrue(flow != fetchFlow);
    Assert.assertEquals(flow.getExecutionId(), fetchFlow.getExecutionId());
    Assert.assertEquals(flow.getEndTime(), fetchFlow.getEndTime());
    Assert.assertEquals(flow.getStartTime(), fetchFlow.getStartTime());
    Assert.assertEquals(flow.getSubmitTime(), fetchFlow.getSubmitTime());
    Assert.assertEquals(flow.getFlowId(), fetchFlow.getFlowId());
    Assert.assertEquals(flow.getProjectId(), fetchFlow.getProjectId());
    Assert.assertEquals(flow.getVersion(), fetchFlow.getVersion());
    Assert.assertEquals(flow.getExecutionOptions().getFailureAction(),
        fetchFlow.getExecutionOptions().getFailureAction());
    Assert.assertEquals(new HashSet<String>(flow.getEndNodes()),
        new HashSet<String>(fetchFlow.getEndNodes()));
  }

  @Test
  public void testUpdateExecutionFlows() throws Exception {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    ExecutableFlow flow = TestUtils.createExecutableFlow("exectest1", "exec1");

    loader.uploadExecutableFlow(flow);

    ExecutableFlow fetchFlow2 =
        loader.fetchExecutableFlow(flow.getExecutionId());

    fetchFlow2.setEndTime(System.currentTimeMillis());
    fetchFlow2.setStatus(Status.SUCCEEDED);
    loader.updateExecutableFlow(fetchFlow2);
    ExecutableFlow fetchFlow =
        loader.fetchExecutableFlow(flow.getExecutionId());

    // Shouldn't be the same object.
    Assert.assertTrue(flow != fetchFlow);
    Assert.assertEquals(flow.getExecutionId(), fetchFlow.getExecutionId());
    Assert.assertEquals(fetchFlow2.getEndTime(), fetchFlow.getEndTime());
    Assert.assertEquals(fetchFlow2.getStatus(), fetchFlow.getStatus());
    Assert.assertEquals(flow.getStartTime(), fetchFlow.getStartTime());
    Assert.assertEquals(flow.getSubmitTime(), fetchFlow.getSubmitTime());
    Assert.assertEquals(flow.getFlowId(), fetchFlow.getFlowId());
    Assert.assertEquals(flow.getProjectId(), fetchFlow.getProjectId());
    Assert.assertEquals(flow.getVersion(), fetchFlow.getVersion());
    Assert.assertEquals(flow.getExecutionOptions().getFailureAction(),
        fetchFlow.getExecutionOptions().getFailureAction());
    Assert.assertEquals(new HashSet<String>(flow.getEndNodes()),
        new HashSet<String>(fetchFlow.getEndNodes()));
  }

  @Test
  public void testUploadExecutableNode() throws Exception {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    ExecutableFlow flow = createExecutableFlow(10, "exec1");
    flow.setExecutionId(10);

    File jobFile = new File(UNIT_BASE_DIR + "/exectest1", "job10.job");
    Props props = new Props(null, jobFile);
    props.put("test", "test2");
    ExecutableNode oldNode = flow.getExecutableNode("job10");
    oldNode.setStartTime(System.currentTimeMillis());
    loader.uploadExecutableNode(oldNode, props);

    ExecutableJobInfo info = loader.fetchJobInfo(10, "job10", 0);
    Assert.assertEquals(flow.getExecutionId(), info.getExecId());
    Assert.assertEquals(flow.getProjectId(), info.getProjectId());
    Assert.assertEquals(flow.getVersion(), info.getVersion());
    Assert.assertEquals(flow.getFlowId(), info.getFlowId());
    Assert.assertEquals(oldNode.getId(), info.getJobId());
    Assert.assertEquals(oldNode.getStatus(), info.getStatus());
    Assert.assertEquals(oldNode.getStartTime(), info.getStartTime());
    Assert.assertEquals("endTime = " + oldNode.getEndTime()
        + " info endTime = " + info.getEndTime(), oldNode.getEndTime(),
        info.getEndTime());

    // Fetch props
    Props outputProps = new Props();
    outputProps.put("hello", "output");
    oldNode.setOutputProps(outputProps);
    oldNode.setEndTime(System.currentTimeMillis());
    loader.updateExecutableNode(oldNode);

    Props fInputProps = loader.fetchExecutionJobInputProps(10, "job10");
    Props fOutputProps = loader.fetchExecutionJobOutputProps(10, "job10");
    Pair<Props, Props> inOutProps = loader.fetchExecutionJobProps(10, "job10");

    Assert.assertEquals(fInputProps.get("test"), "test2");
    Assert.assertEquals(fOutputProps.get("hello"), "output");
    Assert.assertEquals(inOutProps.getFirst().get("test"), "test2");
    Assert.assertEquals(inOutProps.getSecond().get("hello"), "output");

  }

  /* Test exception when unassigning an missing execution */
  @Test
  public void testUnassignExecutorException() throws ExecutorManagerException,
    IOException {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    try {
      loader.unassignExecutor(2);
      Assert.fail("Expecting exception, but didn't get one");
    } catch (ExecutorManagerException ex) {
      System.out.println("Test true");
    }
  }

  /* Test happy case when unassigning executor for a flow execution */
  @Test
  public void testUnassignExecutor() throws ExecutorManagerException,
    IOException {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    String host = "localhost";
    int port = 12345;
    Executor executor = loader.addExecutor(host, port);
    ExecutableFlow flow = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow);
    loader.assignExecutor(executor.getId(), flow.getExecutionId());
    Assert.assertEquals(
      loader.fetchExecutorByExecutionId(flow.getExecutionId()), executor);
    loader.unassignExecutor(flow.getExecutionId());
    Assert.assertEquals(
      loader.fetchExecutorByExecutionId(flow.getExecutionId()), null);
  }

  /* Test exception when assigning a non-existent executor to a flow */
  @Test
  public void testAssignExecutorInvalidExecutor()
    throws ExecutorManagerException, IOException {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    ExecutableFlow flow = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow);
    try {
      loader.assignExecutor(flow.getExecutionId(), 1);
      Assert.fail("Expecting exception, but didn't get one");
    } catch (ExecutorManagerException ex) {
      System.out.println("Test true");
    }
  }

  /* Test exception when assigning an executor to a non-existent flow execution */
  @Test
  public void testAssignExecutorInvalidExecution()
    throws ExecutorManagerException, IOException {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    String host = "localhost";
    int port = 12345;
    Executor executor = loader.addExecutor(host, port);
    try {
      loader.assignExecutor(2, executor.getId());
      Assert.fail("Expecting exception, but didn't get one");
    } catch (ExecutorManagerException ex) {
      System.out.println("Test true");
    }
  }

  /* Test null return when an invalid execution flows */
  @Test
  public void testFetchMissingExecutorByExecution()
    throws ExecutorManagerException, IOException {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    Assert.assertEquals(loader.fetchExecutorByExecutionId(1), null);
  }

  /* Test null return when for a non-dispatched execution */
  @Test
  public void testFetchExecutorByQueuedExecution()
    throws ExecutorManagerException, IOException {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    ExecutableFlow flow = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow);
    Assert.assertEquals(loader.fetchExecutorByExecutionId(flow.getExecutionId()),
      null);
  }

  /* Test happy case when assigning and fetching an executor to a flow execution */
  @Test
  public void testAssignAndFetchExecutor() throws ExecutorManagerException,
    IOException {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    String host = "localhost";
    int port = 12345;
    Executor executor = loader.addExecutor(host, port);
    ExecutableFlow flow = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow);
    loader.assignExecutor(executor.getId(), flow.getExecutionId());
    Assert.assertEquals(loader.fetchExecutorByExecutionId(flow.getExecutionId()),
      executor);
  }

  /* Test fetchQueuedFlows when there are no queued flows */
  @Test
  public void testFetchNoQueuedFlows() throws ExecutorManagerException,
    IOException {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    List<Pair<ExecutionReference, ExecutableFlow>> queuedFlows =
      loader.fetchQueuedFlows();

    // no execution flows at all i.e. no running, completed or queued flows
    Assert.assertTrue(queuedFlows.isEmpty());

    String host = "lcoalhost";
    int port = 12345;
    Executor executor = loader.addExecutor(host, port);

    ExecutableFlow flow = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow);
    loader.assignExecutor(executor.getId(), flow.getExecutionId());
    // only completed flows
    Assert.assertTrue(queuedFlows.isEmpty());

    ExecutableFlow flow2 = TestUtils.createExecutableFlow("exectest1", "exec2");
    loader.uploadExecutableFlow(flow);
    loader.assignExecutor(executor.getId(), flow.getExecutionId());
    ExecutionReference ref = new ExecutionReference(flow2.getExecutionId());
    loader.addActiveExecutableReference(ref);
    // only running and completed flows
    Assert.assertTrue(queuedFlows.isEmpty());
  }

  /* Test fetchQueuedFlows happy case */
  @Test
  public void testFetchQueuedFlows() throws ExecutorManagerException,
    IOException {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    List<Pair<ExecutionReference, ExecutableFlow>> queuedFlows =
      new LinkedList<Pair<ExecutionReference, ExecutableFlow>>();

    ExecutableFlow flow = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow);
    ExecutableFlow flow2 = TestUtils.createExecutableFlow("exectest1", "exec2");
    loader.uploadExecutableFlow(flow2);

    ExecutionReference ref2 = new ExecutionReference(flow2.getExecutionId());
    loader.addActiveExecutableReference(ref2);
    ExecutionReference ref = new ExecutionReference(flow.getExecutionId());
    loader.addActiveExecutableReference(ref);

    List<Pair<ExecutionReference, ExecutableFlow>> fetchedQueuedFlows = loader.fetchQueuedFlows();
    Assert.assertEquals(2, fetchedQueuedFlows.size());
    Pair<ExecutionReference, ExecutableFlow> fetchedFlow1 = fetchedQueuedFlows.get(0);
    Pair<ExecutionReference, ExecutableFlow> fetchedFlow2 = fetchedQueuedFlows.get(1);

    Assert.assertEquals(ref.getExecId(), fetchedFlow1.getFirst().getExecId());
    Assert.assertEquals(flow.getExecutionId(), fetchedFlow1.getSecond().getExecutionId());
    Assert.assertEquals(flow.getFlowId(), fetchedFlow1.getSecond().getFlowId());
    Assert.assertEquals(flow.getProjectId(), fetchedFlow1.getSecond().getProjectId());
    Assert.assertEquals(ref2.getExecId(), fetchedFlow2.getFirst().getExecId());
    Assert.assertEquals(flow2.getExecutionId(), fetchedFlow2.getSecond().getExecutionId());
    Assert.assertEquals(flow2.getFlowId(), fetchedFlow2.getSecond().getFlowId());
    Assert.assertEquals(flow2.getProjectId(), fetchedFlow2.getSecond().getProjectId());
  }

  /* Test all executors fetch from empty executors */
  @Test
  public void testFetchEmptyExecutors() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    List<Executor> executors = loader.fetchAllExecutors();
    Assert.assertEquals(executors.size(), 0);
  }

  /* Test active executors fetch from empty executors */
  @Test
  public void testFetchEmptyActiveExecutors() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    List<Executor> executors = loader.fetchActiveExecutors();
    Assert.assertEquals(executors.size(), 0);
  }

  /* Test missing executor fetch with search by executor id */
  @Test
  public void testFetchMissingExecutorId() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    Executor executor = loader.fetchExecutor(0);
    Assert.assertEquals(executor, null);
  }

  /* Test missing executor fetch with search by host:port */
  @Test
  public void testFetchMissingExecutorHostPort() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    Executor executor = loader.fetchExecutor("localhost", 12345);
    Assert.assertEquals(executor, null);
  }

  /* Test executor events fetch from with no logged executor */
  @Test
  public void testFetchEmptyExecutorEvents() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    Executor executor = new Executor(1, "localhost", 12345, true);
    List<ExecutorLogEvent> executorEvents =
      loader.getExecutorEvents(executor, 5, 0);
    Assert.assertEquals(executorEvents.size(), 0);
  }

  /* Test logging ExecutorEvents */
  @Test
  public void testExecutorEvents() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    int skip = 1;
    User user = new User("testUser");
    Executor executor = new Executor(1, "localhost", 12345, true);
    String message = "My message ";
    EventType[] events =
      { EventType.CREATED, EventType.HOST_UPDATE, EventType.INACTIVATION };

    for (EventType event : events) {
      loader.postExecutorEvent(executor, event, user.getUserId(),
        message + event.getNumVal());
    }

    List<ExecutorLogEvent> eventLogs =
      loader.getExecutorEvents(executor, 10, skip);
    Assert.assertTrue(eventLogs.size() == 2);

    for (int index = 0; index < eventLogs.size(); ++index) {
      ExecutorLogEvent eventLog = eventLogs.get(index);
      Assert.assertEquals(eventLog.getExecutorId(), executor.getId());
      Assert.assertEquals(eventLog.getUser(), user.getUserId());
      Assert.assertEquals(eventLog.getType(), events[index + skip]);
      Assert.assertEquals(eventLog.getMessage(),
        message + events[index + skip].getNumVal());
    }
  }

  /* Test to add duplicate executors */
  @Test
  public void testDuplicateAddExecutor() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    try {
      String host = "localhost";
      int port = 12345;
      loader.addExecutor(host, port);
      loader.addExecutor(host, port);
      Assert.fail("Expecting exception, but didn't get one");
    } catch (ExecutorManagerException ex) {
      System.out.println("Test true");
    }
  }

  /* Test to try update a non-existent executor */
  @Test
  public void testMissingExecutorUpdate() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    try {
      Executor executor = new Executor(1, "localhost", 1234, true);
      loader.updateExecutor(executor);
      Assert.fail("Expecting exception, but didn't get one");
    } catch (ExecutorManagerException ex) {
      System.out.println("Test true");
    }
    clearDB();
  }

  /* Test add & fetch by Id Executors */
  @Test
  public void testSingleExecutorFetchById() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    List<Executor> executors = addTestExecutors(loader);
    for (Executor executor : executors) {
      Executor fetchedExecutor = loader.fetchExecutor(executor.getId());
      Assert.assertEquals(executor, fetchedExecutor);
    }
  }

  /* Test fetch all executors */
  @Test
  public void testFetchAllExecutors() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    List<Executor> executors = addTestExecutors(loader);

    executors.get(0).setActive(false);
    loader.updateExecutor(executors.get(0));

    List<Executor> fetchedExecutors = loader.fetchAllExecutors();
    Assert.assertEquals(executors.size(), fetchedExecutors.size());

    Assert.assertArrayEquals(executors.toArray(), fetchedExecutors.toArray());
  }

  /* Test fetch only active executors */
  @Test
  public void testFetchActiveExecutors() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    List<Executor> executors = addTestExecutors(loader);

    executors.get(0).setActive(false);
    loader.updateExecutor(executors.get(0));

    List<Executor> fetchedExecutors = loader.fetchActiveExecutors();
    Assert.assertEquals(executors.size(), fetchedExecutors.size() + 1);
    executors.remove(0);

    Assert.assertArrayEquals(executors.toArray(), fetchedExecutors.toArray());
  }

  /* Test add & fetch by host:port Executors */
  @Test
  public void testSingleExecutorFetchHostPort() throws Exception {
    if (!isTestSetup()) {
      return;
    }
    ExecutorLoader loader = createLoader();
    List<Executor> executors = addTestExecutors(loader);
    for (Executor executor : executors) {
      Executor fetchedExecutor =
        loader.fetchExecutor(executor.getHost(), executor.getPort());
      Assert.assertEquals(executor, fetchedExecutor);
    }
  }

  /* Helper method used in methods testing jdbc interface for executors table */
  private List<Executor> addTestExecutors(ExecutorLoader loader)
    throws ExecutorManagerException {
    List<Executor> executors = new ArrayList<Executor>();
    executors.add(loader.addExecutor("localhost1", 12345));
    executors.add(loader.addExecutor("localhost2", 12346));
    executors.add(loader.addExecutor("localhost1", 12347));
    return executors;
  }

  /* Test Executor Inactivation */
  @Test
  public void testExecutorInactivation() throws Exception {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    Executor executor = loader.addExecutor("localhost1", 12345);
    Assert.assertTrue(executor.isActive());

    executor.setActive(false);
    loader.updateExecutor(executor);

    Executor fetchedExecutor = loader.fetchExecutor(executor.getId());

    Assert.assertEquals(executor.getHost(), fetchedExecutor.getHost());
    Assert.assertEquals(executor.getId(), fetchedExecutor.getId());
    Assert.assertEquals(executor.getPort(), fetchedExecutor.getPort());
    Assert.assertFalse(fetchedExecutor.isActive());
  }

  /* Test Removing Executor */
  @Test
  public void testRemovingExecutor() throws Exception {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    Executor executor = loader.addExecutor("localhost1", 12345);
    Assert.assertNotNull(executor);
    loader.removeExecutor("localhost1", 12345);
    Executor fetchedExecutor = loader.fetchExecutor("localhost1", 12345);
    Assert.assertNull(fetchedExecutor);
  }

  /* Test Executor reactivation */
  @Test
  public void testExecutorActivation() throws Exception {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    Executor executor = loader.addExecutor("localhost1", 12345);
    Assert.assertTrue(executor.isActive());

    executor.setActive(false);
    loader.updateExecutor(executor);
    Executor fetchedExecutor = loader.fetchExecutor(executor.getId());
    Assert.assertFalse(fetchedExecutor.isActive());

    executor.setActive(true);
    loader.updateExecutor(executor);
    fetchedExecutor = loader.fetchExecutor(executor.getId());

    Assert.assertEquals(executor, fetchedExecutor);
  }

  @Test
  public void testActiveReference() throws Exception {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    ExecutableFlow flow1 = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow1);
    Executor executor = loader.addExecutor("test", 1);
    loader.assignExecutor(executor.getId(), flow1.getExecutionId());
    ExecutionReference ref1 =
        new ExecutionReference(flow1.getExecutionId(), executor);
    loader.addActiveExecutableReference(ref1);

    ExecutableFlow flow2 = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow2);
    loader.assignExecutor(executor.getId(), flow2.getExecutionId());
    ExecutionReference ref2 =
        new ExecutionReference(flow2.getExecutionId(), executor);
    loader.addActiveExecutableReference(ref2);

    ExecutableFlow flow3 = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow3);

    Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows1 =
        loader.fetchActiveFlows();
    ExecutableFlow flow1Result =
        activeFlows1.get(flow1.getExecutionId()).getSecond();
    Assert.assertNotNull(flow1Result);
    Assert.assertTrue(flow1 != flow1Result);
    Assert.assertEquals(flow1.getExecutionId(), flow1Result.getExecutionId());
    Assert.assertEquals(flow1.getEndTime(), flow1Result.getEndTime());
    Assert.assertEquals(flow1.getStartTime(), flow1Result.getStartTime());
    Assert.assertEquals(flow1.getSubmitTime(), flow1Result.getSubmitTime());
    Assert.assertEquals(flow1.getFlowId(), flow1Result.getFlowId());
    Assert.assertEquals(flow1.getProjectId(), flow1Result.getProjectId());
    Assert.assertEquals(flow1.getVersion(), flow1Result.getVersion());
    Assert.assertEquals(flow1.getExecutionOptions().getFailureAction(),
        flow1Result.getExecutionOptions().getFailureAction());

    ExecutableFlow flow1Result2 =
        activeFlows1.get(flow2.getExecutionId()).getSecond();
    Assert.assertNotNull(flow1Result2);
    Assert.assertTrue(flow2 != flow1Result2);
    Assert.assertEquals(flow2.getExecutionId(), flow1Result2.getExecutionId());
    Assert.assertEquals(flow2.getEndTime(), flow1Result2.getEndTime());
    Assert.assertEquals(flow2.getStartTime(), flow1Result2.getStartTime());
    Assert.assertEquals(flow2.getSubmitTime(), flow1Result2.getSubmitTime());
    Assert.assertEquals(flow2.getFlowId(), flow1Result2.getFlowId());
    Assert.assertEquals(flow2.getProjectId(), flow1Result2.getProjectId());
    Assert.assertEquals(flow2.getVersion(), flow1Result2.getVersion());
    Assert.assertEquals(flow2.getExecutionOptions().getFailureAction(),
        flow1Result2.getExecutionOptions().getFailureAction());

    loader.removeActiveExecutableReference(flow2.getExecutionId());
    Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows2 =
        loader.fetchActiveFlows();

    Assert.assertTrue(activeFlows2.containsKey(flow1.getExecutionId()));
    Assert.assertFalse(activeFlows2.containsKey(flow3.getExecutionId()));
    Assert.assertFalse(activeFlows2.containsKey(flow2.getExecutionId()));
  }

  @Test
  public void testFetchActiveFlowByExecId() throws Exception {
    if (!isTestSetup()) {
      return;
    }

    ExecutorLoader loader = createLoader();
    ExecutableFlow flow1 = TestUtils.createExecutableFlow("exectest1", "exec1");
    loader.uploadExecutableFlow(flow1);
    Executor executor = loader.addExecutor("test", 1);
    loader.assignExecutor(executor.getId(), flow1.getExecutionId());
    ExecutionReference ref1 = new ExecutionReference(flow1.getExecutionId(), executor);
    loader.addActiveExecutableReference(ref1);

    Pair<ExecutionReference, ExecutableFlow> activeFlow1 =
        loader.fetchActiveFlowByExecId(flow1.getExecutionId());

    ExecutionReference execRef1 = activeFlow1.getFirst();
    ExecutableFlow execFlow1 = activeFlow1.getSecond();
    Assert.assertNotNull(execRef1);
    Assert.assertEquals(ref1.getExecId(), execRef1.getExecId());
    Assert.assertEquals(ref1.getExecutor(), execRef1.getExecutor());
    Assert.assertNotNull(execFlow1);
    Assert.assertEquals(flow1.getExecutionId(), execFlow1.getExecutionId());
    Assert.assertEquals(flow1.getFlowId(), execFlow1.getFlowId());
    Assert.assertEquals(flow1.getProjectId(), execFlow1.getProjectId());
    Assert.assertEquals(flow1.getVersion(), execFlow1.getVersion());
  }

  @Ignore @Test
  public void testSmallUploadLog() throws ExecutorManagerException {
    File logDir = new File(UNIT_BASE_DIR + "logtest");
    File[] smalllog =
        { new File(logDir, "log1.log"), new File(logDir, "log2.log"),
            new File(logDir, "log3.log") };

    ExecutorLoader loader = createLoader();
    loader.uploadLogFile(1, "smallFiles", 0, smalllog);

    LogData data = loader.fetchLogs(1, "smallFiles", 0, 0, 50000);
    Assert.assertNotNull(data);
    Assert.assertEquals("Logs length is " + data.getLength(), data.getLength(),
        53);

    System.out.println(data.toString());

    LogData data2 = loader.fetchLogs(1, "smallFiles", 0, 10, 20);
    System.out.println(data2.toString());
    Assert.assertNotNull(data2);
    Assert.assertEquals("Logs length is " + data2.getLength(),
        data2.getLength(), 20);

  }

  @Ignore @Test
  public void testLargeUploadLog() throws ExecutorManagerException {
    File logDir = new File(UNIT_BASE_DIR + "logtest");

    // Multiple of 255 for Henry the Eigth
    File[] largelog =
        { new File(logDir, "largeLog1.log"), new File(logDir, "largeLog2.log"),
            new File(logDir, "largeLog3.log") };

    ExecutorLoader loader = createLoader();
    loader.uploadLogFile(1, "largeFiles", 0, largelog);

    LogData logsResult = loader.fetchLogs(1, "largeFiles", 0, 0, 64000);
    Assert.assertNotNull(logsResult);
    Assert.assertEquals("Logs length is " + logsResult.getLength(),
        logsResult.getLength(), 64000);

    LogData logsResult2 = loader.fetchLogs(1, "largeFiles", 0, 1000, 64000);
    Assert.assertNotNull(logsResult2);
    Assert.assertEquals("Logs length is " + logsResult2.getLength(),
        logsResult2.getLength(), 64000);

    LogData logsResult3 = loader.fetchLogs(1, "largeFiles", 0, 330000, 400000);
    Assert.assertNotNull(logsResult3);
    Assert.assertEquals("Logs length is " + logsResult3.getLength(),
        logsResult3.getLength(), 5493);

    LogData logsResult4 = loader.fetchLogs(1, "largeFiles", 0, 340000, 400000);
    Assert.assertNull(logsResult4);

    LogData logsResult5 = loader.fetchLogs(1, "largeFiles", 0, 153600, 204800);
    Assert.assertNotNull(logsResult5);
    Assert.assertEquals("Logs length is " + logsResult5.getLength(),
        logsResult5.getLength(), 181893);

    LogData logsResult6 = loader.fetchLogs(1, "largeFiles", 0, 150000, 250000);
    Assert.assertNotNull(logsResult6);
    Assert.assertEquals("Logs length is " + logsResult6.getLength(),
        logsResult6.getLength(), 185493);
  }

  @SuppressWarnings("static-access")
  @Ignore @Test
  public void testRemoveExecutionLogsByTime() throws ExecutorManagerException,
      IOException, InterruptedException {

    ExecutorLoader loader = createLoader();

    File logDir = new File(UNIT_BASE_DIR + "logtest");

    // Multiple of 255 for Henry the Eigth
    File[] largelog =
        { new File(logDir, "largeLog1.log"), new File(logDir, "largeLog2.log"),
            new File(logDir, "largeLog3.log") };

    DateTime time1 = DateTime.now();
    loader.uploadLogFile(1, "oldlog", 0, largelog);
    // sleep for 5 seconds
    Thread.currentThread().sleep(5000);
    loader.uploadLogFile(2, "newlog", 0, largelog);

    DateTime time2 = time1.plusMillis(2500);

    int count = loader.removeExecutionLogsByTime(time2.getMillis());
    System.out.print("Removed " + count + " records");
    LogData logs = loader.fetchLogs(1, "oldlog", 0, 0, 22222);
    Assert.assertTrue(logs == null);
    logs = loader.fetchLogs(2, "newlog", 0, 0, 22222);
    Assert.assertFalse(logs == null);
  }

  private ExecutableFlow createExecutableFlow(int executionId, String flowName)
    throws IOException {
    ExecutableFlow execFlow =
      TestUtils.createExecutableFlow("exectest1", flowName);
    execFlow.setExecutionId(executionId);
    return execFlow;
  }

  private ExecutorLoader createLoader() {
    Props props = new Props();
    props.put("database.type", "mysql");

    props.put("mysql.host", host);
    props.put("mysql.port", port);
    props.put("mysql.user", user);
    props.put("mysql.database", database);
    props.put("mysql.password", password);
    props.put("mysql.numconnections", numConnections);

    return new JdbcExecutorLoader(props);
  }

  private boolean isTestSetup() {
    if (!testDBExists) {
      System.err.println("Skipping DB test because Db not setup.");
      return false;
    }

    System.out.println("Running DB test because Db setup.");
    return true;
  }

  public static class CountHandler implements ResultSetHandler<Integer> {
    @Override
    public Integer handle(ResultSet rs) throws SQLException {
      int val = 0;
      while (rs.next()) {
        val++;
      }

      return val;
    }
  }
}

upper case. also using duration might be helpful