MigrationTaskService extends StatefulService { private static Operation createPost(URI uri, State currentState) { Operation op = Operation.createPost(uri); return prepareOp(op, currentState); } MigrationTaskService(); static Service createFactory(); @Override void handleStart(Operation startPost); @Override void handlePatch(Operation patchOperation); @Override void handleMaintenance(Operation maintenanceOperation); static final String STAT_NAME_PROCESSED_DOCUMENTS; static final String STAT_NAME_ESTIMATED_TOTAL_SERVICE_COUNT; static final String STAT_NAME_FETCHED_DOCUMENT_COUNT; static final String STAT_NAME_BEFORE_TRANSFORM_COUNT; static final String STAT_NAME_AFTER_TRANSFORM_COUNT; static final String STAT_NAME_DELETE_RETRY_COUNT; static final String STAT_NAME_DELETED_DOCUMENT_COUNT; static final String STAT_NAME_COUNT_QUERY_TIME_DURATION_MICRO; static final String STAT_NAME_RETRIEVAL_OPERATIONS_DURATION_MICRO; static final String STAT_NAME_RETRIEVAL_QUERY_TIME_DURATION_MICRO_FORMAT; static final String FACTORY_LINK; static final Long DEFAULT_MAINTENANCE_INTERVAL_MILLIS; }
@Test public void successCreateTask() throws Throwable { State initState = validMigrationState(); Operation post = Operation.createPost(this.sourceFactoryUri).setBody(initState); State result = this.sender.sendAndWait(post, State.class); assertNotNull(result); assertEquals(initState.destinationFactoryLink, result.destinationFactoryLink); assertEquals(initState.destinationNodeGroupReference, result.destinationNodeGroupReference); assertEquals(initState.sourceFactoryLink, result.sourceFactoryLink); assertEquals(initState.sourceNodeGroupReference, result.sourceNodeGroupReference); }
@Test public void failMigrationWhenSourceFactoryNotAvailable() throws Throwable { clearSourceAndDestInProcessPeers = true; URI factoryUri = UriUtils.buildUri(getSourceHost(), ExampleService.FACTORY_LINK); this.host.waitForReplicatedFactoryServiceAvailable(factoryUri); for (VerificationHost h : this.host.getInProcessHostMap().values()) { Utils.setFactoryAvailabilityIfOwner(h, ExampleService.FACTORY_LINK, ServiceUriPaths.DEFAULT_NODE_SELECTOR, false); } waitForReplicatedFactoryServiceNotAvailable(getSourceHost(), factoryUri, ServiceUriPaths.DEFAULT_NODE_SELECTOR); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); Operation op = Operation.createPost(this.destinationFactoryUri) .setBody(migrationState) .setReferer(this.host.getUri()); String selfLink = getDestinationHost().getTestRequestSender().sendAndWait(op) .getBody(State.class).documentSelfLink; State waitForServiceCompletion = waitForServiceCompletion(selfLink, getDestinationHost()); assertEquals(waitForServiceCompletion.taskInfo.stage, TaskStage.FAILED); assertTrue("Invalid message received: " + waitForServiceCompletion.taskInfo.failure.message, waitForServiceCompletion.taskInfo.failure.message.contains( "Failed to verify availability of source factory service")); }
@Test public void successMigrateDocumentsCustomNodeGroupWithObserver() throws Throwable { List<ExampleServiceState> states = createExampleDocuments(this.exampleWithCustomSelectorSourceFactory, getSourceHost(), this.serviceCount); MigrationTaskService.State migrationState = validMigrationStateForCustomNodeGroup(); Operation op = this.sender.sendAndWait( Operation.createPost(this.destinationFactoryUri).setBody(migrationState)); String taskLink = op.getBody(State.class).documentSelfLink; State finalServiceState = waitForServiceCompletion(taskLink, getDestinationHost()); assertEquals(TaskStage.FINISHED, finalServiceState.taskInfo.stage); ServiceDocumentQueryResult res = getDestinationHost() .getFactoryState(this.exampleWithCustomSelectorDestinationFactoryOnObserver); assertEquals(0L, (long) res.documentCount); assertTrue(res.documentLinks.isEmpty()); URI customExampleFactoryOnPeer = this.exampleWithCustomSelectorDestinationFactory; res = getDestinationHost().getFactoryState(customExampleFactoryOnPeer); assertEquals(states.size(), (long) res.documentCount); assertEquals(states.size(), res.documentLinks.size()); }
@Test public void successNoDocuments() throws Throwable { MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); Operation post = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); ServiceDocument taskState = this.sender.sendAndWait(post, ServiceDocument.class); State finalState = waitForServiceCompletion(taskState.documentSelfLink, getDestinationHost()); assertEquals(TaskStage.FINISHED, finalState.taskInfo.stage); ServiceStats stats = this.sender.sendStatsGetAndWait(UriUtils.buildUri(getDestinationHost(), taskState.documentSelfLink)); assertFalse(stats.entries.containsKey(MigrationTaskService.STAT_NAME_PROCESSED_DOCUMENTS)); }
@Test public void successMigrateMultiPageResult() throws Throwable { List<ExampleServiceState> states = createExampleDocuments(this.exampleSourceFactory, getSourceHost(), this.serviceCount); MigrationTaskService.State migrationState = validMigrationState( ExampleService.FACTORY_LINK); QuerySpecification spec = new QuerySpecification(); spec.resultLimit = (int) (this.serviceCount / 10); migrationState.querySpec = spec; TestContext ctx = testCreate(1); String[] out = new String[1]; Operation op = Operation.createPost(this.destinationFactoryUri) .setBody(migrationState) .setCompletion((o, e) -> { if (e != null) { this.host.log("Post service error: %s", Utils.toString(e)); ctx.failIteration(e); return; } out[0] = o.getBody(State.class).documentSelfLink; ctx.completeIteration(); }); getDestinationHost().send(op); testWait(ctx); State waitForServiceCompletion = waitForServiceCompletion(out[0], getDestinationHost()); ServiceStats stats = getStats(out[0], getDestinationHost()); Long processedDocuments = Long.valueOf((long) stats.entries.get(MigrationTaskService.STAT_NAME_PROCESSED_DOCUMENTS).latestValue); assertEquals(TaskStage.FINISHED, waitForServiceCompletion.taskInfo.stage); assertEquals(Long.valueOf(this.serviceCount), processedDocuments); List<URI> uris = getFullUri(getDestinationHost(), states); getDestinationHost().getServiceState(EnumSet.noneOf(TestProperty.class), ExampleServiceState.class, uris); }
@Test public void successMigrateSameDocumentsTwice() throws Throwable { List<ExampleServiceState> states = createExampleDocuments(this.exampleSourceFactory, getSourceHost(), this.serviceCount); MigrationTaskService.State migrationState = validMigrationState( ExampleService.FACTORY_LINK); TestContext ctx = testCreate(1); String[] out = new String[1]; Operation op = Operation.createPost(this.destinationFactoryUri) .setBody(migrationState) .setCompletion((o, e) -> { if (e != null) { this.host.log("Post service error: %s", Utils.toString(e)); ctx.failIteration(e); return; } out[0] = o.getBody(State.class).documentSelfLink; ctx.completeIteration(); }); getDestinationHost().send(op); testWait(ctx); State waitForServiceCompletion = waitForServiceCompletion(out[0], getDestinationHost()); ServiceStats stats = getStats(out[0], getDestinationHost()); Long processedDocuments = Long.valueOf((long) stats.entries.get(MigrationTaskService.STAT_NAME_PROCESSED_DOCUMENTS).latestValue); assertEquals(TaskStage.FINISHED, waitForServiceCompletion.taskInfo.stage); assertEquals(Long.valueOf(this.serviceCount), processedDocuments); TestContext ctx2 = testCreate(1); migrationState.documentSelfLink = null; op = Operation.createPost(this.destinationFactoryUri) .setBody(migrationState) .setCompletion((o, e) -> { if (e != null) { this.host.log("Post service error: %s", Utils.toString(e)); ctx2.failIteration(e); return; } out[0] = o.getBody(State.class).documentSelfLink; ctx2.completeIteration(); }); getDestinationHost().send(op); testWait(ctx2); waitForServiceCompletion = waitForServiceCompletion(out[0], getDestinationHost()); stats = getStats(out[0], getDestinationHost()); assertEquals(waitForServiceCompletion.taskInfo.stage, TaskStage.FINISHED); processedDocuments = Long.valueOf((long) stats.entries.get(MigrationTaskService.STAT_NAME_PROCESSED_DOCUMENTS).latestValue); assertEquals(Long.valueOf(this.serviceCount), processedDocuments); List<URI> uris = getFullUri(getDestinationHost(), states); getDestinationHost().getServiceState(EnumSet.noneOf(TestProperty.class), ExampleServiceState.class, uris); }
@Test public void migrationWithDocumentOwnerMismatch() throws Throwable { clearSourceAndDestInProcessPeers = true; int peersWithDocs; int iteration = 0; do { ++iteration; createExampleDocuments(this.exampleSourceFactory, getSourceHost(), this.serviceCount, false); ServiceDocumentQueryResult results = this.host.getExpandedFactoryState(this.exampleSourceFactory); peersWithDocs = results.documents.values().stream() .map(doc -> (Utils.fromJson(doc, ServiceDocument.class)).documentOwner) .distinct() .collect(Collectors.toList()) .size(); } while (peersWithDocs != this.nodeCount); Iterator<VerificationHost> peerIt = this.host.getInProcessHostMap().values().iterator(); VerificationHost peerNode = null; while (peerIt.hasNext()) { VerificationHost h = peerIt.next(); if (!h.isOwner(ExampleService.FACTORY_LINK, ServiceUriPaths.DEFAULT_NODE_SELECTOR)) { peerNode = h; } h.setPeerSynchronizationEnabled(false); } assertNotNull(peerNode); this.host.setNodeGroupQuorum(this.nodeCount - 1); this.host.stopHost(peerNode); this.host.waitForNodeGroupConvergence(this.nodeCount - 1); MigrationTaskService.State migrationState = validMigrationState( ExampleService.FACTORY_LINK); String[] out = new String[1]; Operation op = Operation.createPost(this.destinationFactoryUri) .setBody(migrationState) .setReferer(this.host.getUri()); out[0] = getDestinationHost().getTestRequestSender().sendAndWait(op).getBody(State.class).documentSelfLink; State waitForServiceCompletion = waitForServiceCompletion(out[0], getDestinationHost()); assertEquals(TaskStage.FINISHED, waitForServiceCompletion.taskInfo.stage); ServiceDocumentQueryResult result = this.host .getFactoryState(UriUtils.buildUri( getDestinationHost(), ExampleService.FACTORY_LINK)); assertEquals(this.serviceCount * iteration, result.documentLinks.size()); }
@Test public void successMigrateSameDocumentsTwiceUsingFallback() throws Throwable { for (VerificationHost host : destinationHost.getInProcessHostMap().values()) { host.toggleServiceOptions(UriUtils.buildUri(host, ExampleService.FACTORY_LINK), null, EnumSet.of(ServiceOption.IDEMPOTENT_POST)); } List<ExampleServiceState> states = createExampleDocuments(this.exampleSourceFactory, getSourceHost(), this.serviceCount); MigrationTaskService.State migrationState = validMigrationState( ExampleService.FACTORY_LINK); migrationState.migrationOptions = EnumSet.of(MigrationOption.DELETE_AFTER); Operation op = Operation.createPost(this.destinationFactoryUri) .setBody(migrationState); op = this.sender.sendAndWait(op); String migrationTaskLink = op.getBody(State.class).documentSelfLink; State waitForServiceCompletion = waitForServiceCompletion(migrationTaskLink, getDestinationHost()); ServiceStats stats = getStats(migrationTaskLink, getDestinationHost()); Long processedDocuments = Long.valueOf((long) stats.entries.get( MigrationTaskService.STAT_NAME_PROCESSED_DOCUMENTS).latestValue); assertEquals(waitForServiceCompletion.taskInfo.stage, TaskStage.FINISHED); assertEquals(Long.valueOf(this.serviceCount), processedDocuments); for (int i = 0; i < this.iterationCount; i++) { this.host.log("Start migration with pre-existing target documents (%d)", i); migrationState.documentSelfLink = null; op = Operation.createPost(this.destinationFactoryUri) .setBody(migrationState); op = this.sender.sendAndWait(op); migrationTaskLink = op.getBody(State.class).documentSelfLink; this.host.log("Created task %s", migrationTaskLink); waitForServiceCompletion = waitForServiceCompletion(migrationTaskLink, getDestinationHost()); assertEquals(TaskStage.FINISHED, waitForServiceCompletion.taskInfo.stage); stats = getStats(migrationTaskLink, getDestinationHost()); processedDocuments = Long.valueOf((long) stats.entries .get(MigrationTaskService.STAT_NAME_PROCESSED_DOCUMENTS).latestValue); assertEquals(Long.valueOf(this.serviceCount), processedDocuments); stats = getStats(migrationTaskLink, getDestinationHost()); processedDocuments = Long.valueOf((long) stats.entries .get(MigrationTaskService.STAT_NAME_PROCESSED_DOCUMENTS).latestValue); } List<URI> uris = getFullUri(getDestinationHost(), states); getDestinationHost().getServiceState(EnumSet.noneOf(TestProperty.class), ExampleServiceState.class, uris); }
@Test public void successTaskRestartedByMaintenance() throws Throwable { List<ExampleServiceState> states = createExampleDocuments(this.exampleSourceFactory, getSourceHost(), this.serviceCount); long expectedUpdateTime = states.stream().map(state -> state.documentUpdateTimeMicros).max(Long::compareTo).get(); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.taskInfo = TaskState.createAsCancelled(); migrationState.continuousMigration = Boolean.TRUE; getDestinationHost().setMaintenanceIntervalMicros(migrationState.maintenanceIntervalMicros / 10); Operation op = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); State state = this.sender.sendAndWait(op, State.class); Set<TaskStage> finalStages = new HashSet<>(Arrays.asList(TaskStage.FAILED, TaskStage.FINISHED)); State finalServiceState = waitForServiceCompletion(state.documentSelfLink, getDestinationHost(), finalStages); assertEquals(TaskStage.FINISHED, finalServiceState.taskInfo.stage); ServiceStats stats = getStats(state.documentSelfLink, getDestinationHost()); long processedDocuments = (long) stats.entries.get(MigrationTaskService.STAT_NAME_PROCESSED_DOCUMENTS).latestValue; boolean processed = this.serviceCount <= processedDocuments; assertTrue(format("%d <= %d", this.serviceCount, processedDocuments), processed); assertEquals(Long.valueOf(expectedUpdateTime), finalServiceState.latestSourceUpdateTimeMicros); List<URI> uris = getFullUri(getDestinationHost(), states); getDestinationHost().getServiceState(EnumSet.noneOf(TestProperty.class), ExampleServiceState.class, uris); }
@Test public void failOnSourceNodeFailureBeforeIssuingQuery() throws Throwable { MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.sourceNodeGroupReference = FAKE_URI; migrationState.destinationNodeGroupReference = UriUtils.extendUri(getDestinationHost().getPublicUri(), ServiceUriPaths.DEFAULT_NODE_GROUP); Operation op = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); State state = this.sender.sendAndWait(op, State.class); State waitForServiceCompletion = waitForServiceCompletion(state.documentSelfLink, getDestinationHost()); assertEquals(TaskStage.FAILED, waitForServiceCompletion.taskInfo.stage); }
@Test public void failOnDestinationNodeFailureBeforeIssuingQuery() throws Throwable { createExampleDocuments(this.exampleSourceFactory, getSourceHost(), this.serviceCount); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.sourceNodeGroupReference = UriUtils.extendUri(getSourceHost().getPublicUri(), ServiceUriPaths.DEFAULT_NODE_GROUP); migrationState.destinationNodeGroupReference = FAKE_URI; Operation op = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); State state = this.sender.sendAndWait(op, State.class); State waitForServiceCompletion = waitForServiceCompletion(state.documentSelfLink, getDestinationHost()); assertEquals(TaskStage.FAILED, waitForServiceCompletion.taskInfo.stage); }
@Test public void successMigrateDocumentsWithAllVersions() throws Throwable { clearSourceAndDestInProcessPeers = true; Map<String, DocumentVersionType> typeBySelfLink = createVersionedExampleDocuments(this.exampleSourceFactory, this.serviceCount); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.migrationOptions = EnumSet.of(MigrationOption.ALL_VERSIONS); Operation post = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); ServiceDocument taskState = this.sender.sendAndWait(post, ServiceDocument.class); State finalState = waitForServiceCompletion(taskState.documentSelfLink, getDestinationHost()); assertEquals(TaskStage.FINISHED, finalState.taskInfo.stage); validateVersionedExampleDocuments(typeBySelfLink); long expectedProcessedDocs = this.serviceCount * (1 + 2 + 3 + 1 + 2 + 3); validateNumOfProcessedDoc(getDestinationHost(), taskState.documentSelfLink, expectedProcessedDocs); }
@Test public void failOnMigrateDocumentsWithAllVersionsWithoutDeleteAfter() throws Throwable { clearSourceAndDestInProcessPeers = true; for (VerificationHost host : destinationHost.getInProcessHostMap().values()) { host.toggleServiceOptions(UriUtils.buildUri(host, ExampleService.FACTORY_LINK), null, EnumSet.of(ServiceOption.IDEMPOTENT_POST)); } ExampleServiceState doc = new ExampleService.ExampleServiceState(); doc.name = "foo" ; doc.documentSelfLink = "foo"; this.sender.sendAndWait(Operation.createPost(this.exampleSourceFactory).setBody(doc)); this.sender.sendAndWait(Operation.createPost(this.exampleDestinationFactory).setBody(doc)); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.migrationOptions = EnumSet.of(MigrationOption.ALL_VERSIONS); Operation post = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); State taskState = this.sender.sendAndWait(post, State.class); State finalState = waitForServiceCompletion(taskState.documentSelfLink, getDestinationHost()); assertEquals(TaskStage.FAILED, finalState.taskInfo.stage); }
@Test public void successMigrateDocumentsWithAllVersionsWithDeleteAfter() throws Throwable { clearSourceAndDestInProcessPeers = true; for (VerificationHost host : destinationHost.getInProcessHostMap().values()) { host.toggleServiceOptions(UriUtils.buildUri(host, ExampleService.FACTORY_LINK), null, EnumSet.of(ServiceOption.IDEMPOTENT_POST)); } Map<String, DocumentVersionType> typeBySelfLink = createVersionedExampleDocuments(this.exampleSourceFactory, this.serviceCount); List<String> selfLinks = typeBySelfLink.entrySet().stream() .filter(entry -> entry.getValue() != DocumentVersionType.POST_PUT_PATCH_DELETE) .map(Entry::getKey) .collect(toList()); List<Operation> posts = new ArrayList<>(); for (String selfLink : selfLinks) { ExampleServiceState state = new ExampleServiceState(); state.name = "example-in-destination"; state.documentSelfLink = selfLink; Operation post = Operation.createPost(getDestinationHost(), ExampleService.FACTORY_LINK).setBody(state); posts.add(post); } this.sender.sendAndWait(posts); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.migrationOptions = EnumSet.of(MigrationOption.ALL_VERSIONS, MigrationOption.DELETE_AFTER); Operation post = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); ServiceDocument taskState = this.sender.sendAndWait(post, ServiceDocument.class); State finalState = waitForServiceCompletion(taskState.documentSelfLink, getDestinationHost()); assertEquals(TaskStage.FINISHED, finalState.taskInfo.stage); validateVersionedExampleDocuments(typeBySelfLink); long expectedProcessedDocs = this.serviceCount * (1 + 2 + 3 + 1 + 2 + 3); validateNumOfProcessedDoc(getDestinationHost(), taskState.documentSelfLink, expectedProcessedDocs); }
@Test public void successMigrateWithCalculateEstimateOption() throws Throwable { createExampleDocuments(this.exampleSourceFactory, getSourceHost(), this.serviceCount); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.migrationOptions = EnumSet.of(MigrationOption.ESTIMATE_COUNT); Operation post = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); ServiceDocument taskState = this.sender.sendAndWait(post, ServiceDocument.class); State finalState = waitForServiceCompletion(taskState.documentSelfLink, getDestinationHost()); assertEquals(TaskStage.FINISHED, finalState.taskInfo.stage); ServiceStats stats = this.sender.sendStatsGetAndWait(UriUtils.buildUri(getDestinationHost(), taskState.documentSelfLink)); ServiceStat statValue = stats.entries.get(MigrationTaskService.STAT_NAME_ESTIMATED_TOTAL_SERVICE_COUNT); assertNotNull("estimatedTotalServiceCount should be populated", statValue); assertEquals("estimatedTotalServiceCount", this.serviceCount, (long) statValue.latestValue); }
@Test public void failNodeGroupReferenceValidation() throws Throwable { URI sourceUri = this.host.getPeerHost().getUri(); URI destUri = getDestinationHost().getUri(); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.sourceReferences = Collections.singletonList(sourceUri); Operation op = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); FailureResponse response = this.sender.sendAndWaitFailure(op); assertEquals(Operation.STATUS_CODE_BAD_REQUEST, response.op.getStatusCode()); migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.sourceNodeGroupReference = null; op = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); response = this.sender.sendAndWaitFailure(op); assertEquals(Operation.STATUS_CODE_BAD_REQUEST, response.op.getStatusCode()); migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.destinationReferences = Collections.singletonList(destUri); op = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); response = this.sender.sendAndWaitFailure(op); assertEquals(Operation.STATUS_CODE_BAD_REQUEST, response.op.getStatusCode()); migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.destinationFactoryLink = null; op = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); response = this.sender.sendAndWaitFailure(op); assertEquals(Operation.STATUS_CODE_BAD_REQUEST, response.op.getStatusCode()); }
@Test public void successMigrateWithSourceUrisAndDestinationUris() throws Throwable { List<ExampleServiceState> states = createExampleDocuments(this.exampleSourceFactory, getSourceHost(), this.serviceCount); List<URI> sourceUris = new ArrayList<>(this.host.getNodeGroupMap().values()); List<URI> destUris = new ArrayList<>(destinationHost.getNodeGroupMap().values()); MigrationTaskService.State migrationState = validMigrationState(ExampleService.FACTORY_LINK); migrationState.sourceNodeGroupReference = null; migrationState.destinationNodeGroupReference = null; migrationState.sourceReferences = sourceUris; migrationState.destinationReferences = destUris; Operation op = Operation.createPost(this.destinationFactoryUri).setBody(migrationState); State state = this.sender.sendAndWait(op, State.class); State finalServiceState = waitForServiceCompletion(state.documentSelfLink, getDestinationHost()); assertEquals(TaskStage.FINISHED, finalServiceState.taskInfo.stage); List<URI> uris = getFullUri(getDestinationHost(), states); this.sender.sendAndWait(uris.stream().map(Operation::createGet).collect(toList())); }