NodeStateChangeChecker { public Result evaluateTransition( Node node, ClusterState clusterState, SetUnitStateRequest.Condition condition, NodeState oldState, NodeState newState) { if (condition == SetUnitStateRequest.Condition.FORCE) { return Result.allowSettingOfWantedState(); } if (condition != SetUnitStateRequest.Condition.SAFE) { return Result.createDisallowed("Condition not implemented: " + condition.name()); } if (node.getType() != NodeType.STORAGE) { return Result.createDisallowed("Safe-set of node state is only supported for storage nodes! " + "Requested node type: " + node.getType().toString()); } NodeInfo nodeInfo = clusterInfo.getStorageNodeInfo(node.getIndex()); if (nodeInfo == null) { return Result.createDisallowed("Unknown node " + node); } if (newState.getState().equals(oldState.getState())) { return Result.createAlreadySet(); } switch (newState.getState()) { case UP: return canSetStateUp(nodeInfo); case MAINTENANCE: return canSetStateMaintenanceTemporarily(node, clusterState); case DOWN: return canSetStateDownPermanently(nodeInfo, clusterState); default: return Result.createDisallowed("Destination node state unsupported in safe mode: " + newState); } } NodeStateChangeChecker(
            int minStorageNodesUp,
            double minRatioOfStorageNodesUp,
            int requiredRedundancy,
            ClusterInfo clusterInfo); Result evaluateTransition(
            Node node, ClusterState clusterState, SetUnitStateRequest.Condition condition,
            NodeState oldState, NodeState newState); static final String BUCKETS_METRIC_NAME; static final Map<String, String> BUCKETS_METRIC_DIMENSIONS; }
@Test public void testCanUpgradeForce() { NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(createCluster(createNodes(1))); NodeState newState = new NodeState(NodeType.STORAGE, State.INITIALIZING); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeDistributor, defaultAllUpClusterState(), SetUnitStateRequest.Condition.FORCE, UP_NODE_STATE, newState); assertTrue(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); }
@Test public void testUnknownStorageNode() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = new NodeStateChangeChecker( 5 , minRatioOfStorageNodesUp, requiredRedundancy, cluster.clusterInfo()); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( new Node(NodeType.STORAGE, 10), defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), is("Unknown node storage.10")); }
@Test public void testSafeSetStateDistributors() { NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(createCluster(createNodes(1))); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeDistributor, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), containsString("Safe-set of node state is only supported for storage nodes")); }
@Test public void testCanUpgradeSafeMissingStorage() { ContentCluster cluster = createCluster(createNodes(4)); setAllNodesUp(cluster, HostInfo.createHostInfo(createDistributorHostInfo(4, 5, 6))); NodeStateChangeChecker nodeStateChangeChecker = new NodeStateChangeChecker( 5 , minRatioOfStorageNodesUp, requiredRedundancy, cluster.clusterInfo()); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), is("There are only 4 storage nodes up, while config requires at least 5")); }
@Test public void testSetUpFailsIfReportedIsDown() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, MAINTENANCE_NODE_STATE, UP_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); }
@Test public void testSetUpSucceedsIfReportedIsUpButGeneratedIsDown() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); markAllNodesAsReportingStateUp(cluster); ClusterState stateWithNodeDown = clusterState(String.format( "version:%d distributor:4 storage:4 .%d.s:d", currentClusterStateVersion, nodeStorage.getIndex())); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, stateWithNodeDown, SetUnitStateRequest.Condition.SAFE, MAINTENANCE_NODE_STATE, UP_NODE_STATE); assertTrue(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); }
@Test public void testCanSetUpEvenIfOldWantedStateIsDown() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); setAllNodesUp(cluster, HostInfo.createHostInfo(createDistributorHostInfo(4, 3, 6))); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, new NodeState(NodeType.STORAGE, State.DOWN), UP_NODE_STATE); assertTrue(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); }
@Test public void testCanUpgradeStorageSafeNo() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); setAllNodesUp(cluster, HostInfo.createHostInfo(createDistributorHostInfo(4, 3, 6))); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), is("Distributor 0 says storage node 1 " + "has buckets with redundancy as low as 3, but we require at least 4")); }
@Test public void testCanUpgradeIfMissingMinReplicationFactor() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); setAllNodesUp(cluster, HostInfo.createHostInfo(createDistributorHostInfo(4, 3, 6))); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( new Node(NodeType.STORAGE, 3), defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertTrue(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); }
@Test public void testCanUpgradeIfStorageNodeMissingFromNodeInfo() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); String hostInfo = "{\n" + " \"cluster-state-version\": 2,\n" + " \"distributor\": {\n" + " \"storage-nodes\": [\n" + " {\n" + " \"node-index\": 0,\n" + " \"min-current-replication-factor\": " + requiredRedundancy + "\n" + " }\n" + " ]\n" + " }\n" + "}\n"; setAllNodesUp(cluster, HostInfo.createHostInfo(hostInfo)); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( new Node(NodeType.STORAGE, 1), defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertTrue(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); }
@Test public void testMissingDistributorState() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); cluster.clusterInfo().getStorageNodeInfo(1).setReportedState(new NodeState(NodeType.STORAGE, State.UP), 0); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); assertThat(result.getReason(), is("Distributor node (0) has not reported any cluster state version yet.")); }
@Test public void testNodeRatioRequirementConsidersGeneratedNodeStates() { ContentCluster cluster = createCluster(createNodes(4)); NodeStateChangeChecker nodeStateChangeChecker = createChangeChecker(cluster); markAllNodesAsReportingStateUp(cluster); ClusterState stateWithNodeDown = clusterState(String.format( "version:%d distributor:4 storage:4 .3.s:d", currentClusterStateVersion)); NodeStateChangeChecker.Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, stateWithNodeDown, SetUnitStateRequest.Condition.SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); assertFalse(result.settingWantedStateIsAllowed()); assertFalse(result.wantedStateAlreadySet()); }