NodeGroupService extends StatefulService { @Override public URI getUri() { if (this.uri == null) { this.uri = super.getUri(); } return this.uri; } NodeGroupService(); @Override URI getUri(); @Override void handleStart(Operation startPost); @Override void handleGet(Operation get); @Override void handlePatch(Operation patch); @RouteDocumentation( description = "Join a peer to this node group", requestBodyType = JoinPeerRequest.class) @Override void handlePost(Operation post); @Override void handleMaintenance(Operation maint); void handleGossipPatchCompletion(long sendTimeMicros, Operation maint, Operation patch,
            Throwable e,
            NodeGroupState localState, NodeGroupState patchBody, AtomicInteger remaining,
            NodeState remotePeer); static final String STAT_NAME_JOIN_RETRY_COUNT; static final long PEER_REQUEST_TIMEOUT_MICROS; static final int MIN_PEER_GOSSIP_COUNT; static final String STAT_NAME_RESTARTING_SERVICES_COUNT; static final String STAT_NAME_RESTARTING_SERVICES_FAILURE_COUNT; static final String STAT_NAME_PREFIX_GOSSIP_PATCH_DURATION; }
@Test public void synchronizationRequestQueuing() throws Throwable { setUp(this.nodeCount); this.host.joinNodesAndVerifyConvergence(this.host.getPeerCount()); this.host.setNodeGroupQuorum(this.nodeCount); waitForReplicatedFactoryServiceAvailable( this.host.getPeerServiceUri(ExampleService.FACTORY_LINK), ServiceUriPaths.DEFAULT_NODE_SELECTOR); waitForReplicationFactoryConvergence(); VerificationHost peerHost = this.host.getPeerHost(); List<URI> exampleUris = this.host.createExampleServices(peerHost, 1, null); URI instanceUri = exampleUris.get(0); ExampleServiceState synchState = new ExampleServiceState(); synchState.documentSelfLink = UriUtils.getLastPathSegment(instanceUri); TestContext ctx = this.host.testCreate(this.updateCount); for (int i = 0; i < this.updateCount; i++) { Operation op = Operation.createPost(peerHost, ExampleService.FACTORY_LINK) .setBody(synchState) .addPragmaDirective(Operation.PRAGMA_DIRECTIVE_SYNCH_OWNER) .setReferer(this.host.getUri()) .setCompletion(ctx.getCompletion()); this.host.sendRequest(op); } ctx.await(); synchState.documentSelfLink = "unknown-service"; TestContext notFoundCtx = this.host.testCreate(1); Operation notFoundOp = Operation .createPost(peerHost, ExampleService.FACTORY_LINK) .setBody(synchState) .addPragmaDirective(Operation.PRAGMA_DIRECTIVE_SYNCH_OWNER) .setReferer(this.host.getUri()) .setCompletion((o, e) -> { if (e != null && o.getStatusCode() == Operation.STATUS_CODE_NOT_FOUND) { notFoundCtx.completeIteration(); return; } notFoundCtx.failIteration( new IllegalStateException("Failure expected with 404 NOT FOUND error")); }); this.host.sendRequest(notFoundOp); notFoundCtx.await(); }
@Test public void replicationWithOutOfOrderPostAndUpdates() throws Throwable { setUp(this.nodeCount); this.host.joinNodesAndVerifyConvergence(this.host.getPeerCount()); this.host.setNodeGroupQuorum(this.nodeCount / 2 + 1); waitForReplicatedFactoryServiceAvailable( this.host.getPeerServiceUri(ExampleService.FACTORY_LINK), ServiceUriPaths.DEFAULT_NODE_SELECTOR); waitForReplicationFactoryConvergence(); ExampleServiceState state = new ExampleServiceState(); state.name = "testing"; state.counter = 1L; VerificationHost peer = this.host.getPeerHost(); Map<String, ExampleServiceState> exampleStatesPerSelfLink = new HashMap<>(); TestContext ctx = this.host.testCreate(this.serviceCount * this.updateCount); for (int i = 0; i < this.serviceCount; i++) { Operation post = Operation .createPost(peer, ExampleService.FACTORY_LINK) .setBody(state) .setReferer(this.host.getUri()) .setCompletion((o, e) -> { if (e != null) { ctx.failIteration(e); return; } ExampleServiceState rsp = o.getBody(ExampleServiceState.class); for (int k = 0; k < this.updateCount; k++) { ExampleServiceState update = new ExampleServiceState(); state.counter = (long) k; Operation patch = Operation .createPatch(peer, rsp.documentSelfLink) .setBody(update) .setReferer(this.host.getUri()) .setCompletion((op, ex) -> { if (ex != null) { ctx.failIteration(ex); return; } ExampleServiceState updatedState = op.getBody(ExampleServiceState.class); exampleStatesPerSelfLink.put(rsp.documentSelfLink, updatedState); ctx.completeIteration(); }); this.host.sendRequest(patch); } }); this.host.sendRequest(post); } ctx.await(); this.host.waitForReplicatedFactoryChildServiceConvergence( getFactoriesPerNodeGroup(ExampleService.FACTORY_LINK), exampleStatesPerSelfLink, this.exampleStateConvergenceChecker, exampleStatesPerSelfLink.size(), this.updateCount, this.replicationFactor); }
@Test public void replicationInMemoryWithOutOfOrderPostAndUpdates() throws Throwable { setUp(this.nodeCount); this.host.joinNodesAndVerifyConvergence(this.host.getPeerCount()); this.host.setNodeGroupQuorum(this.nodeCount / 2 + 1); for (VerificationHost h : this.host.getInProcessHostMap().values()) { h.startServiceAndWait(ExampleNonPersistedService.createFactory(), ExampleNonPersistedService.FACTORY_LINK, null); } ExampleServiceState state = new ExampleServiceState(); state.name = "testing"; state.counter = 1L; VerificationHost peer = this.host.getPeerHost(); TestContext ctx = this.host.testCreate(this.serviceCount * this.updateCount); for (int i = 0; i < this.serviceCount; i++) { Operation post = Operation .createPost(peer, ExampleNonPersistedService.FACTORY_LINK) .setBody(state) .setReferer(this.host.getUri()) .setCompletion((o, e) -> { if (e != null) { ctx.failIteration(e); return; } ExampleServiceState rsp = o.getBody(ExampleServiceState.class); for (int k = 0; k < this.updateCount; k++) { ExampleServiceState update = new ExampleServiceState(); state.counter = (long) k; Operation patch = Operation .createPatch(peer, rsp.documentSelfLink) .setBody(update) .setReferer(this.host.getUri()) .setCompletion(ctx.getCompletion()); this.host.sendRequest(patch); } }); this.host.sendRequest(post); } ctx.await(); }
@Test public void synchronizationServiceNotFoundOnNewOwner() throws Throwable { this.isPeerSynchronizationEnabled = false; setUp(this.nodeCount); this.host.joinNodesAndVerifyConvergence(this.host.getPeerCount()); this.host.setNodeGroupQuorum(this.nodeCount); this.host.waitForNodeGroupConvergence(this.nodeCount); VerificationHost peer = this.host.getPeerHost(); List<URI> exampleUris = this.host.createExampleServices(peer, this.serviceCount * 5, null); List<ServiceHost> inMemoryHosts = new ArrayList<>(); this.host.getInProcessHostMap().values().forEach(h -> inMemoryHosts.add(h)); this.host.testStart(1); VerificationHost newHost = this.host.setUpLocalPeerHost( 0, peer.getMaintenanceIntervalMicros(), inMemoryHosts); this.host.testWait(); List<URI> peerUris = new ArrayList<>(); peerUris.add(peer.getUri()); newHost.joinPeers(peerUris, ServiceUriPaths.DEFAULT_NODE_GROUP); this.host.setNodeGroupQuorum(this.nodeCount + 1); this.host.waitForNodeGroupConvergence(this.nodeCount + 1); List<URI> filteredUris = exampleUris.stream() .filter(u -> newHost.isOwner(u.getPath(), this.replicationNodeSelector)) .collect(Collectors.toList()); if (filteredUris.size() == 0) { this.host.log("New host does not own any service."); return; } if (filteredUris.size() > 1) { TestContext ctx = this.host.testCreate(filteredUris.size() - 1); int i = 0; for (; i < filteredUris.size() / 2; i++) { URI exampleUri = filteredUris.get(i); ExampleServiceState state = new ExampleServiceState(); state.name = UUID.randomUUID().toString(); state.counter = 100L; Operation patchOp = Operation .createPatch(exampleUri) .setBody(state) .setReferer(this.host.getUri()) .setCompletion(ctx.getCompletion()); this.host.send(patchOp); } for (; i < filteredUris.size(); i++) { URI exampleUri = UriUtils.buildStatsUri(filteredUris.get(i)); Operation patchOp = Operation .createGet(exampleUri) .setReferer(this.host.getUri()) .setCompletion(ctx.getCompletion()); this.host.send(patchOp); } ctx.await(); } }
@Test public void synchronizationAfterStaleHostRestart() throws Throwable { this.isPeerSynchronizationEnabled = false; setUp(this.nodeCount); this.host.joinNodesAndVerifyConvergence(this.nodeCount); VerificationHost peerHost = this.host.getPeerHost(); Map<String, ExampleServiceState> exampleStatesPerSelfLink = createExampleServices(peerHost.getUri()); this.host.setNodeGroupQuorum(this.nodeCount - 1); NodeGroupConfig cfg = new NodeGroupConfig(); cfg.nodeRemovalDelayMicros = TimeUnit.SECONDS.toMicros(1); this.host.setNodeGroupConfig(cfg); this.host.stopHostAndPreserveState(peerHost); this.host.waitForNodeGroupConvergence(this.nodeCount - 1, this.nodeCount - 1); VerificationHost existingHost = this.host.getPeerHost(); Map<String, ExampleServiceState> servicesToUpdate = new HashMap<>(); boolean update = false; for (Entry<String, ExampleServiceState> e : exampleStatesPerSelfLink.entrySet()) { update = !update; if (update) { servicesToUpdate.put(e.getKey(), e.getValue()); } } doExampleServicePatch(servicesToUpdate, existingHost.getUri()); this.host.setNodeGroupQuorum(this.nodeCount); peerHost.setPort(0); peerHost.setSecurePort(0); peerHost.setPeerSynchronizationEnabled(false); assertTrue(VerificationHost.restartStatefulHost(peerHost, false)); this.host.addPeerNode(peerHost); this.host.joinNodesAndVerifyConvergence(this.nodeCount); this.host.setNodeGroupQuorum(this.nodeCount); this.host.waitForNodeGroupConvergence(); VerificationHost owner = this.host.getOwnerPeer(ExampleService.FACTORY_LINK); this.host.waitFor("wait node selector available timeout", () -> { Operation op = Operation.createGet(owner, ServiceUriPaths.DEFAULT_NODE_SELECTOR); NodeSelectorState nss = this.host.getTestRequestSender().sendAndWait(op, NodeSelectorState.class); if (nss.status == NodeSelectorState.Status.AVAILABLE) { return true; } return false; }); startSynchronizationTaskAndWait(owner, ExampleService.FACTORY_LINK, ExampleServiceState.class, 1L); this.host.waitForReplicatedFactoryChildServiceConvergence( getFactoriesPerNodeGroup(ExampleService.FACTORY_LINK), exampleStatesPerSelfLink, this.exampleStateConvergenceChecker, exampleStatesPerSelfLink.size(), 0, this.replicationFactor); List<String> nonComparedLinks = new ArrayList<>(this.serviceCount); HashMap<String, ServiceDocument> services = new HashMap<>(this.serviceCount); for (VerificationHost h : this.host.getInProcessHostMap().values()) { ServiceDocumentQueryResult r = this.host.getFactoryState( UriUtils.buildExpandLinksQueryUri(UriUtils.buildUri(h, ExampleService.FACTORY_LINK))); for (Entry<String, Object> e : r.documents.entrySet()) { ServiceDocument newDoc = Utils.fromJson(e.getValue(), ServiceDocument.class); ServiceDocument prevDoc = services.get(e.getKey()); if (prevDoc == null) { services.put(e.getKey(), newDoc); nonComparedLinks.add(e.getKey()); continue; } assertTrue(newDoc.documentVersion == prevDoc.documentVersion && newDoc.documentEpoch == prevDoc.documentEpoch); nonComparedLinks.remove(e.getKey()); } } assertTrue(nonComparedLinks.isEmpty()); URI queryTaskUri = UriUtils.buildUri(peerHost, ServiceUriPaths.CORE_LOCAL_QUERY_TASKS); long actualTotalVersionCount = 0; for (String selfLink : services.keySet()) { Query query = Query.Builder.create() .addFieldClause(ServiceDocument.FIELD_NAME_SELF_LINK, selfLink) .build(); QueryTask queryTask = QueryTask.Builder.createDirectTask() .setQuery(query) .addOption(QueryOption.INCLUDE_ALL_VERSIONS) .addOption(QueryOption.COUNT) .build(); Operation postQuery = Operation.createPost(queryTaskUri) .setBody(queryTask); QueryTask queryResponse = peerHost.getTestRequestSender().sendAndWait(postQuery, QueryTask.class); actualTotalVersionCount += queryResponse.results.documentCount.longValue(); } long expectedTotalVersionCount = exampleStatesPerSelfLink.size() + servicesToUpdate.size(); assertEquals(expectedTotalVersionCount, actualTotalVersionCount); }
@Test public void synchronizationCollisionWithPosts() throws Throwable { setUp(this.nodeCount); this.host.joinNodesAndVerifyConvergence(this.host.getPeerCount()); this.host.setNodeGroupQuorum(this.nodeCount); this.host.waitForNodeGroupConvergence(this.nodeCount); URI factoryUri = UriUtils.buildUri(this.host.getPeerHost(), ExampleService.FACTORY_LINK); waitForReplicatedFactoryServiceAvailable(factoryUri, this.replicationNodeSelector); String taskPath = UriUtils.buildUriPath( SynchronizationTaskService.FACTORY_LINK, UriUtils.convertPathCharsFromLink(ExampleService.FACTORY_LINK)); VerificationHost owner = this.host.getOwnerPeer(ExampleService.FACTORY_LINK); this.host.log(Level.INFO, "Owner of synch-task is %s", owner.getId()); URI taskUri = UriUtils.buildUri(owner, taskPath); SynchronizationTaskService.State taskState = this.host.getServiceState( null, SynchronizationTaskService.State.class, taskUri); long membershipUpdateTimeMicros = taskState.membershipUpdateTimeMicros; ExampleServiceState state = new ExampleServiceState(); state.name = "testing"; TestContext ctx = this.host.testCreate(this.serviceCount * 10); for (int i = 0; i < this.serviceCount * 10; i++) { if (i == 5) { startSynchronizationTaskAndWait(owner, ExampleService.FACTORY_LINK, ExampleServiceState.class, membershipUpdateTimeMicros + 1); } Operation post = Operation .createPost(factoryUri) .setBody(state) .setReferer(this.host.getUri()) .setCompletion(ctx.getCompletion()); this.host.sendRequest(post); } ctx.await(); }
@Test public void recognizeSelfInPeerNodesByPublicUri() throws Throwable { String id = "node-" + VerificationHost.hostNumber.incrementAndGet(); String publicUri = "http: ExampleServiceHost nodeA = new ExampleServiceHost() { @Override public List<URI> getInitialPeerHosts() { try { Field field = ServiceHost.class.getDeclaredField("state"); field.setAccessible(true); ServiceHostState s = (ServiceHostState) field.get(this); s.initialPeerNodes = new String[] { publicUri + getPort() }; s.publicUri = URI.create(publicUri + getPort()); } catch (Exception e) { throw new RuntimeException(e); } return super.getInitialPeerHosts(); } }; TemporaryFolder tmpFolderA = new TemporaryFolder(); tmpFolderA.create(); try { String[] args = { "--port=0", "--id=" + id, "--publicUri=" + publicUri, "--bindAddress=127.0.0.1", "--sandbox=" + tmpFolderA.getRoot().getAbsolutePath() }; nodeA.initialize(args); nodeA.setMaintenanceIntervalMicros(TimeUnit.MILLISECONDS .toMicros(VerificationHost.FAST_MAINT_INTERVAL_MILLIS)); nodeA.start(); URI nodeGroupUri = UriUtils.buildUri(nodeA, ServiceUriPaths.DEFAULT_NODE_GROUP, null); TestRequestSender sender = new TestRequestSender(nodeA); Operation op = Operation.createGet(nodeGroupUri) .setReferer(nodeA.getUri()); NodeGroupState nodeGroupState = sender.sendAndWait(op, NodeGroupState.class); assertEquals(1, nodeGroupState.nodes.size()); assertEquals(1, nodeGroupState.nodes.values().iterator().next().membershipQuorum); } finally { tmpFolderA.delete(); nodeA.stop(); } }
@Test public void testMultiNodeNonReplicatedExpiration() throws Throwable { setUp(this.nodeCount); this.host.joinNodesAndVerifyConvergence(this.host.getPeerCount()); this.host.setNodeGroupQuorum(this.nodeCount); this.host.waitForNodeGroupConvergence(this.nodeCount); this.replicationTargetFactoryLink = NonPersistedExampleFactoryService.SELF_LINK; for (VerificationHost h : this.host.getInProcessHostMap().values()) { h.startServiceAndWait(NonPersistedExampleFactoryService.class, NonPersistedExampleFactoryService.SELF_LINK); } for (URI hostUri : this.host.getNodeGroupMap().keySet()) { waitForReplicatedFactoryServiceAvailable( UriUtils.buildUri(hostUri, this.replicationTargetFactoryLink), ServiceUriPaths.DEFAULT_NODE_SELECTOR); } VerificationHost targetHost = this.host.getPeerHost(); URI targetHostUri = targetHost.getUri(); Map<String, ExampleServiceState> initialStates = createExampleServices(targetHostUri); this.host.log(Level.INFO, "Waiting for %d services to replicate", initialStates.size()); for (URI hostUri : this.host.getNodeGroupMap().keySet()) { URI factoryUri = UriUtils.extendUri(hostUri, this.replicationTargetFactoryLink); this.host.waitFor("Services failed to replicate", () -> { ServiceDocumentQueryResult result = this.host.getServiceState( EnumSet.noneOf(TestProperty.class), ServiceDocumentQueryResult.class, factoryUri); return result.documentLinks.size() == initialStates.size(); }); } long expirationMicros = Utils.fromNowMicrosUtc(TimeUnit.MILLISECONDS.toMicros( this.host.maintenanceIntervalMillis)); ExampleServiceState patchBody = new ExampleServiceState(); patchBody.documentExpirationTimeMicros = expirationMicros; this.host.log(Level.INFO, "Patching expiration time on %d services", initialStates.size()); TestContext ctx = this.host.testCreate(initialStates.size()); for (String selfLink : initialStates.keySet()) { Operation patchOp = Operation.createPatch(targetHost, selfLink) .setBody(patchBody) .setReferer(this.host.getReferer()) .setCompletion(ctx.getCompletion()); this.host.send(patchOp); } ctx.await(); for (URI hostUri : this.host.getNodeGroupMap().keySet()) { URI factoryUri = UriUtils.extendUri(hostUri, this.replicationTargetFactoryLink); this.host.waitFor("Services failed to expire", () -> { ServiceDocumentQueryResult result = this.host.getServiceState( EnumSet.noneOf(TestProperty.class), ServiceDocumentQueryResult.class, factoryUri); this.host.log(Level.INFO, "Query response: %s", Utils.toJsonHtml(result)); return result.documentLinks.size() == 0; }); } }
@Test public void synchronizationAllVersions() throws Throwable { TestXenonConfiguration.override( SynchronizationTaskService.class, "SYNCH_ALL_VERSIONS", "true" ); this.isPeerSynchronizationEnabled = false; setUp(this.nodeCount); this.host.joinNodesAndVerifyConvergence(this.nodeCount); this.host.setNodeGroupQuorum(this.nodeCount - 1); NodeGroupConfig cfg = new NodeGroupConfig(); cfg.nodeRemovalDelayMicros = TimeUnit.SECONDS.toMicros(1); this.host.setNodeGroupConfig(cfg); VerificationHost stoppedHost = this.host.getPeerHost(); this.host.stopHostAndPreserveState(stoppedHost); this.host.waitForNodeGroupConvergence(this.nodeCount - 1, this.nodeCount - 1); VerificationHost peerHost = this.host.getPeerHost(); URI peerHostUri = peerHost.getUri(); Map<String, ExampleServiceState> exampleStatesPerSelfLink = createExampleServices( peerHostUri); doExampleServicePatch(exampleStatesPerSelfLink, peerHostUri); this.host.setNodeGroupQuorum(this.nodeCount); stoppedHost.setPort(0); stoppedHost.setSecurePort(0); stoppedHost.setPeerSynchronizationEnabled(false); assertTrue(VerificationHost.restartStatefulHost(stoppedHost, false)); this.host.addPeerNode(stoppedHost); this.host.joinNodesAndVerifyConvergence(this.nodeCount); this.host.setNodeGroupQuorum(this.nodeCount); this.host.waitForNodeGroupConvergence(); VerificationHost owner = this.host.getOwnerPeer(ExampleService.FACTORY_LINK); this.host.waitFor("wait node selector available timeout", () -> { Operation op = Operation.createGet(owner, ServiceUriPaths.DEFAULT_NODE_SELECTOR); NodeSelectorState nss = this.host.getTestRequestSender().sendAndWait(op, NodeSelectorState.class); if (nss.status == NodeSelectorState.Status.AVAILABLE) { return true; } return false; }); startSynchronizationTaskAndWait(owner, ExampleService.FACTORY_LINK, ExampleServiceState.class, 1L); this.host.waitForReplicatedFactoryChildServiceConvergence( getFactoriesPerNodeGroup(ExampleService.FACTORY_LINK), exampleStatesPerSelfLink, this.exampleStateConvergenceChecker, exampleStatesPerSelfLink.size(), this.updateCount, this.replicationFactor); verifySameVersionsAcrossHosts(true); }
@Test public void synchronizationWithPeerNodeListAndDuplicates() throws Throwable { VerificationHost h = null; TemporaryFolder tmpFolder = new TemporaryFolder(); tmpFolder.create(); try { setUp(this.nodeCount); this.host.setNodeGroupQuorum(1); Map<String, ExampleServiceState> exampleStatesPerSelfLink = new HashMap<>(); int dupServiceCount = this.serviceCount; AtomicInteger counter = new AtomicInteger(); Map<URI, ExampleServiceState> dupStates = new HashMap<>(); for (VerificationHost v : this.host.getInProcessHostMap().values()) { counter.set(0); URI factoryUri = UriUtils.buildFactoryUri(v, ExampleService.class); dupStates = this.host.doFactoryChildServiceStart( null, dupServiceCount, ExampleServiceState.class, (o) -> { ExampleServiceState s = new ExampleServiceState(); s.documentSelfLink = "duplicateExampleInstance-" + counter.incrementAndGet(); s.name = s.documentSelfLink; o.setBody(s); }, factoryUri); } for (ExampleServiceState s : dupStates.values()) { exampleStatesPerSelfLink.put(s.documentSelfLink, s); } this.serviceCount = exampleStatesPerSelfLink.size(); Collection<URI> peerNodeGroupUris = new ArrayList<>(); StringBuilder peerNodes = new StringBuilder(); for (VerificationHost peer : this.host.getInProcessHostMap().values()) { peerNodeGroupUris.add(UriUtils.buildUri(peer, ServiceUriPaths.DEFAULT_NODE_GROUP)); peerNodes.append(peer.getUri().toString()).append(","); } for (URI nodeGroup : this.host.getNodeGroupMap().values()) { this.host.subscribeForNodeGroupConvergence(nodeGroup, this.nodeCount + 1, (o, e) -> { if (e != null) { this.host.log("Error in notification: %s", Utils.toString(e)); } }); } h = new VerificationHost(); int quorum = this.host.getPeerCount() + 1; String mainHostId = "main-" + VerificationHost.hostNumber.incrementAndGet(); String[] args = { "--port=0", "--id=" + mainHostId, "--bindAddress=127.0.0.1", "--sandbox=" + tmpFolder.getRoot().getAbsolutePath(), "--peerNodes=" + peerNodes.toString() }; h.initialize(args); h.setPeerSynchronizationEnabled(this.isPeerSynchronizationEnabled); h.setMaintenanceIntervalMicros(TimeUnit.MILLISECONDS .toMicros(VerificationHost.FAST_MAINT_INTERVAL_MILLIS)); h.start(); URI mainHostNodeGroupUri = UriUtils.buildUri(h, ServiceUriPaths.DEFAULT_NODE_GROUP); int totalCount = this.nodeCount + 1; peerNodeGroupUris.add(mainHostNodeGroupUri); this.host.waitForNodeGroupIsAvailableConvergence(); this.host.waitForNodeGroupConvergence(peerNodeGroupUris, totalCount, totalCount, true); this.host.setNodeGroupQuorum(quorum, mainHostNodeGroupUri); this.host.setNodeGroupQuorum(quorum); this.host.scheduleSynchronizationIfAutoSyncDisabled(this.replicationNodeSelector); int peerNodeCount = h.getInitialPeerHosts().size(); assertTrue(totalCount >= peerNodeCount + 1); verifyReplicatedInConflictPost(dupStates); this.host.addPeerNode(h); this.host.waitForReplicatedFactoryChildServiceConvergence( getFactoriesPerNodeGroup(this.replicationTargetFactoryLink), exampleStatesPerSelfLink, this.exampleStateConvergenceChecker, this.serviceCount, 0, this.replicationFactor); doStateUpdateReplicationTest(Action.PATCH, this.serviceCount, this.updateCount, 0, this.exampleStateUpdateBodySetter, this.exampleStateConvergenceChecker, exampleStatesPerSelfLink); URI exampleFactoryUri = this.host.getPeerServiceUri(ExampleService.FACTORY_LINK); waitForReplicatedFactoryServiceAvailable( UriUtils.buildUri(exampleFactoryUri, ExampleService.FACTORY_LINK), ServiceUriPaths.DEFAULT_NODE_SELECTOR); } finally { this.host.log("test finished"); if (h != null) { h.stop(); tmpFolder.delete(); } } }
@Test public void nodeRestartWithSameAddressDifferentId() throws Throwable { int failedNodeCount = 1; int afterFailureQuorum = this.nodeCount - failedNodeCount; setUp(this.nodeCount); setOperationTimeoutMicros(TimeUnit.SECONDS.toMicros(5)); this.host.joinNodesAndVerifyConvergence(this.nodeCount); this.host.log("Stopping node"); this.host.setNodeGroupQuorum(afterFailureQuorum); List<ServiceHostState> hostStates = stopHostsToSimulateFailure(failedNodeCount); URI remainingPeerNodeGroup = this.host.getPeerNodeGroupUri(); this.host.waitForNodeGroupConvergence(this.nodeCount - failedNodeCount); ServiceHostState stoppedHostState = hostStates.get(0); this.host.testStart(1); VerificationHost newHost = this.host.setUpLocalPeerHost(stoppedHostState.httpPort, VerificationHost.FAST_MAINT_INTERVAL_MILLIS, null); this.host.testWait(); URI newHostNodeGroupService = UriUtils .buildUri(newHost.getUri(), ServiceUriPaths.DEFAULT_NODE_GROUP); this.host.testStart(1); this.host.joinNodeGroup(newHostNodeGroupService, remainingPeerNodeGroup); this.host.testWait(); this.host.waitForNodeGroupConvergence(this.nodeCount); }