LastRevRecoveryAgent { public int recover(int clusterId, long waitUntil) throws DocumentStoreException { ClusterNodeInfoDocument nodeInfo = missingLastRevUtil.getClusterNodeInfo(clusterId); if (nodeInfo != null) { if (nodeInfo.isRecoveryNeeded(revisionContext.getClock().getTime())) { NodeDocument root = missingLastRevUtil.getRoot(); Revision lastRev = root.getLastRev().get(clusterId); Revision sweepRev = root.getSweepRevisions().getRevision(clusterId); long startTime; String reason; if (lastRev != null) { startTime = lastRev.getTimestamp(); reason = "lastRev: " + lastRev.toString(); } else { startTime = nodeInfo.getStartTime(); reason = String.format( "no lastRev for root, using startTime %d", startTime); } if (sweepRev != null && sweepRev.getTimestamp() < startTime) { startTime = sweepRev.getTimestamp(); reason = "sweepRev: " + sweepRev.toString(); } Revision bc = getEarliestBranchCommitMergedAround(root, startTime, clusterId); if (bc != null) { startTime = bc.getTimestamp(); reason = "branchRev: " + bc.toString(); } return recoverCandidates(nodeInfo, startTime, waitUntil, reason); } } log.debug("No recovery needed for clusterId {}", clusterId); return 0; } LastRevRecoveryAgent(DocumentStore store,
                                RevisionContext revisionContext,
                                MissingLastRevSeeker seeker,
                                Consumer<Integer> afterRecovery);  LastRevRecoveryAgent(DocumentStore store, RevisionContext context); int recover(int clusterId, long waitUntil); int recover(int clusterId); int recover(Iterable<NodeDocument> suspects, int clusterId); int recover(final Iterable<NodeDocument> suspects,
                       final int clusterId, final boolean dryRun); boolean isRecoveryNeeded(); void performRecoveryIfNeeded(); Iterable<Integer> getRecoveryCandidateNodes();  }
@Test public void recoveryOfModifiedDocument() throws Exception { ds1.setMaxBackOffMillis(0); ds2.setMaxBackOffMillis(0); NodeBuilder b1 = ds1.getRoot().builder(); b1.child("x").child("y").setProperty("p", "v1"); merge(ds1, b1); ds1.runBackgroundOperations(); ds2.runBackgroundOperations(); NodeBuilder b2 = ds2.getRoot().builder(); b2.child("x").child("y").setProperty("p", "v2"); merge(ds2, b2); long leaseTime = ds2.getClusterInfo().getLeaseTime(); clock.waitUntil(clock.getTime() + leaseTime * 2); b1 = ds1.getRoot().builder(); b1.child("x").child("y").setProperty("p", "v11"); try { merge(ds1, b1); fail("CommitFailedException expected"); } catch (CommitFailedException e) { } ds1.getLastRevRecoveryAgent().recover(2); ds1.runBackgroundOperations(); b1 = ds1.getRoot().builder(); b1.child("x").child("y").setProperty("p", "v11"); merge(ds1, b1); disposeQuietly(ds2); }