LuceneDocumentIndexService extends StatelessService { @Override public URI getUri() { return this.uri; } LuceneDocumentIndexService();  LuceneDocumentIndexService(String indexDirectory); static void setImplicitQueryResultLimit(int limit); static int getImplicitQueryResultLimit(); static void setImplicitQueryProcessingPageSize(int limit); static int getImplicitQueryProcessingPageSize(); static void setIndexFileCountThresholdForWriterRefresh(int count); static int getIndexFileCountThresholdForWriterRefresh(); static void setExpiredDocumentSearchThreshold(int count); static int getExpiredDocumentSearchThreshold(); static void setVersionRetentionBulkCleanupThreshold(int count); static int getVersionRetentionBulkCleanupThreshold(); static void setVersionRetentionServiceThreshold(int count); static int getVersionRetentionServiceThreshold(); static long getSearcherRefreshIntervalMicros(); static void setSearcherRefreshIntervalMicros(long interval); static void setMetadataUpdateMaxQueueDepth(int depth); static int getMetadataUpdateMaxQueueDepth(); @Override void handleStart(final Operation post); IndexWriter createWriter(File directory, boolean doUpgrade); @Override void authorizeRequest(Operation op); @Override void handleRequest(Operation op); void handleGetImpl(Operation get); void handleDeleteImpl(Operation delete); @Override URI getUri(); @Override void handleMaintenance(Operation post); static final String SELF_LINK; static final int QUERY_THREAD_COUNT; static final int UPDATE_THREAD_COUNT; static final int QUERY_QUEUE_DEPTH; static final int UPDATE_QUEUE_DEPTH; static final String FILE_PATH_LUCENE; static final int DEFAULT_INDEX_FILE_COUNT_THRESHOLD_FOR_WRITER_REFRESH; static final int DEFAULT_INDEX_SEARCHER_COUNT_THRESHOLD; static final int MIN_QUERY_RESULT_LIMIT; static final int DEFAULT_QUERY_RESULT_LIMIT; static final int DEFAULT_QUERY_PAGE_RESULT_LIMIT; static final int DEFAULT_EXPIRED_DOCUMENT_SEARCH_THRESHOLD; static final int DEFAULT_METADATA_UPDATE_MAX_QUEUE_DEPTH; static final long DEFAULT_PAGINATED_SEARCHER_EXPIRATION_DELAY; static final String STAT_NAME_ACTIVE_QUERY_FILTERS; static final String STAT_NAME_ACTIVE_PAGINATED_QUERIES; static final String STAT_NAME_COMMIT_COUNT; static final String STAT_NAME_INDEX_LOAD_RETRY_COUNT; static final String STAT_NAME_COMMIT_DURATION_MICROS; static final String STAT_NAME_GROUP_QUERY_COUNT; static final String STAT_NAME_QUERY_DURATION_MICROS; static final String STAT_NAME_GROUP_QUERY_DURATION_MICROS; static final String STAT_NAME_QUERY_SINGLE_DURATION_MICROS; static final String STAT_NAME_QUERY_ALL_VERSIONS_DURATION_MICROS; static final String STAT_NAME_RESULT_PROCESSING_DURATION_MICROS; static final String STAT_NAME_INDEXED_FIELD_COUNT; static final String STAT_NAME_INDEXED_DOCUMENT_COUNT; static final String STAT_NAME_FORCED_UPDATE_DOCUMENT_DELETE_COUNT; static final String STAT_NAME_FIELD_COUNT_PER_DOCUMENT; static final String STAT_NAME_INDEXING_DURATION_MICROS; static final String STAT_NAME_SEARCHER_UPDATE_COUNT; static final String STAT_NAME_SEARCHER_REUSE_BY_DOCUMENT_KIND_COUNT; static final String STAT_NAME_PAGINATED_SEARCHER_UPDATE_COUNT; static final String STAT_NAME_PAGINATED_SEARCHER_FORCE_DELETION_COUNT; static final String STAT_NAME_WRITER_ALREADY_CLOSED_EXCEPTION_COUNT; static final String STAT_NAME_READER_ALREADY_CLOSED_EXCEPTION_COUNT; static final String STAT_NAME_SERVICE_DELETE_COUNT; static final String STAT_NAME_DOCUMENT_EXPIRATION_COUNT; static final String STAT_NAME_DOCUMENT_EXPIRATION_FORCED_MAINTENANCE_COUNT; static final String STAT_NAME_METADATA_INDEXING_UPDATE_COUNT; static final String STAT_NAME_VERSION_CACHE_LOOKUP_COUNT; static final String STAT_NAME_VERSION_CACHE_MISS_COUNT; static final String STAT_NAME_VERSION_CACHE_ENTRY_COUNT; static final String STAT_NAME_MAINTENANCE_SEARCHER_REFRESH_DURATION_MICROS; static final String STAT_NAME_MAINTENANCE_DOCUMENT_EXPIRATION_DURATION_MICROS; static final String STAT_NAME_MAINTENANCE_VERSION_RETENTION_DURATION_MICROS; static final String STAT_NAME_MAINTENANCE_METADATA_INDEXING_DURATION_MICROS; static final String STAT_NAME_DOCUMENT_KIND_QUERY_COUNT_FORMAT; static final String STAT_NAME_NON_DOCUMENT_KIND_QUERY_COUNT; static final String STAT_NAME_SINGLE_QUERY_BY_FACTORY_COUNT_FORMAT; static final String STAT_NAME_PREFIX_UPDATE_QUEUE_DEPTH; static final String STAT_NAME_FORMAT_UPDATE_QUEUE_DEPTH; static final String STAT_NAME_PREFIX_QUERY_QUEUE_DEPTH; static final String STAT_NAME_FORMAT_QUERY_QUEUE_DEPTH; }
@Test public void documentGetStats() throws Throwable { setUpHost(false); List<String> docLinks = new ArrayList<String>(); MinimalFactoryTestService f = new MinimalFactoryTestService(); MinimalFactoryTestService factoryService = (MinimalFactoryTestService) this.host .startServiceAndWait(f, UUID.randomUUID().toString(), null); factoryService.setChildServiceCaps( EnumSet.of(ServiceOption.PERSISTENCE)); for (int i = 0; i < this.serviceCount; i++) { MinimalTestServiceState initialState = new MinimalTestServiceState(); initialState.documentSelfLink = initialState.id = UUID.randomUUID().toString(); this.host.sendAndWaitExpectSuccess( Operation.createPost(factoryService.getUri()).setBody(initialState)); docLinks.add(initialState.documentSelfLink); } for (String selfLink: docLinks) { this.host.sendAndWaitExpectSuccess(Operation.createGet(this.host, UriUtils.buildUriPath(factoryService.getUri().getPath(), selfLink))); } Map<String, ServiceStat> stats = this.host.getServiceStats(this.host.getDocumentIndexServiceUri()); String statKey = String.format( LuceneDocumentIndexService.STAT_NAME_SINGLE_QUERY_BY_FACTORY_COUNT_FORMAT, factoryService.getUri().getPath()); ServiceStat kindStat = stats.get(statKey); assertNotNull(kindStat); assertEquals(this.serviceCount, kindStat.latestValue, 0); }
@Test public void testQueueDepthStats() throws Throwable { setUpHost(false); MinimalFactoryTestService factoryService = (MinimalFactoryTestService) this.host.startServiceAndWait(new MinimalFactoryTestService(), UUID.randomUUID().toString(), null); factoryService.setChildServiceCaps(EnumSet.of(ServiceOption.PERSISTENCE)); this.host.doFactoryChildServiceStart(null, this.serviceCount, MinimalTestServiceState.class, (o) -> { MinimalTestServiceState initialState = new MinimalTestServiceState(); initialState.documentSelfLink = initialState.id = UUID.randomUUID().toString(); o.setBody(initialState); }, factoryService.getUri()); ExecutorService blockingExecutor = Executors.newSingleThreadExecutor((r) -> null); ExecutorService queryExecutor = this.indexService.setQueryExecutorService(blockingExecutor); queryExecutor.shutdown(); queryExecutor.awaitTermination(this.host.getTimeoutSeconds(), TimeUnit.SECONDS); QueryTask queryTask = QueryTask.Builder.create() .setQuery(Query.Builder.create() .addKindFieldClause(MinimalTestServiceState.class) .build()) .build(); this.host.createQueryTaskService(queryTask); this.host.waitFor("Query queue depth stat was not set", () -> { Map<String, ServiceStat> indexStats = this.host.getServiceStats( this.host.getDocumentIndexServiceUri()); ServiceStat queueDepthStat = indexStats.entrySet().stream() .filter((entry) -> entry.getKey().startsWith( LuceneDocumentIndexService.STAT_NAME_PREFIX_QUERY_QUEUE_DEPTH)) .filter((entry) -> entry.getKey().endsWith( ServiceStats.STAT_NAME_SUFFIX_PER_DAY)) .map(Entry::getValue).findFirst().orElse(null); if (queueDepthStat == null) { return false; } assertTrue(queueDepthStat.name.contains(ServiceUriPaths.CORE_AUTHZ_GUEST_USER)); return queueDepthStat.latestValue == 1; }); }
@Test public void immutableServiceLifecycle() throws Throwable { setUpHost(false); URI factoryUri = createImmutableFactoryService(this.host); doThroughputPostWithNoQueryResults(false, factoryUri); ServiceDocumentQueryResult res = this.host.getFactoryState(factoryUri); assertEquals(this.serviceCount, res.documentLinks.size()); TestContext ctx = this.host.testCreate(res.documentLinks.size()); ctx.setTestName("DELETE").logBefore(); for (String link : res.documentLinks) { Operation del = Operation.createDelete(this.host, link).setCompletion((o, e) -> { ctx.completeIteration(); }); this.host.send(del); } ctx.await(); ctx.logAfter(); MinimalFactoryTestService f = new MinimalFactoryTestService(); MinimalFactoryTestService factoryService = (MinimalFactoryTestService) this.host .startServiceAndWait(f, UUID.randomUUID().toString(), null); factoryService.setChildServiceCaps( EnumSet.of(ServiceOption.PERSISTENCE, ServiceOption.IMMUTABLE)); MinimalTestServiceState body = new MinimalTestServiceState(); body.id = "id"; Operation post = Operation.createPost(factoryService.getUri()).setBody(body); this.host.sendAndWaitExpectSuccess(post); post = Operation.createPost(factoryService.getUri()).setBody(body); factoryService.setChildServiceCaps( EnumSet.of(ServiceOption.PERSISTENCE, ServiceOption.IMMUTABLE, ServiceOption.PERIODIC_MAINTENANCE)); this.host.sendAndWaitExpectFailure(post); post = Operation.createPost(factoryService.getUri()).setBody(body); factoryService.setChildServiceCaps( EnumSet.of(ServiceOption.PERSISTENCE, ServiceOption.IMMUTABLE, ServiceOption.INSTRUMENTATION)); this.host.sendAndWaitExpectFailure(post); }
@Test public void updateAndQueryByVersion() throws Throwable { setUpHost(false); this.host.doExampleServiceUpdateAndQueryByVersion(this.host.getUri(), (int) this.serviceCount); }
@Test public void putWithFailureAndCacheValidation() throws Throwable { setUpHost(false); List<Service> services = this.host.doThroughputServiceStart( 1, MinimalTestService.class, this.host.buildMinimalTestState(), EnumSet.of(Service.ServiceOption.PERSISTENCE), null); URI uri = services.get(0).getUri(); MinimalTestServiceState largeBody = new MinimalTestServiceState(); Random r = new Random(); byte[] data = new byte[ServiceDocumentDescription.DEFAULT_SERIALIZED_STATE_LIMIT * 2]; r.nextBytes(data); largeBody.id = printBase64Binary(data); this.host.testStart(1); Operation put = Operation .createPut(uri) .setBody(largeBody) .setCompletion( (o, e) -> { if (e == null) { this.host.failIteration(new IllegalStateException( "Request should have failed")); return; } ServiceErrorResponse rsp = o.getErrorResponseBody(); if (!rsp.message.contains("size limit")) { this.host.failIteration(new IllegalStateException( "Error message not expected")); return; } this.host.completeIteration(); }); this.host.send(put); this.host.testWait(); this.host.doServiceUpdates(Action.PUT, 1, EnumSet.of(TestProperty.LARGE_PAYLOAD, TestProperty.FORCE_FAILURE), services); Map<URI, MinimalTestServiceState> states = this.host.getServiceState(null, MinimalTestServiceState.class, services); for (MinimalTestServiceState s : states.values()) { if (s.documentVersion > 0) { throw new IllegalStateException("version should have not incremented"); } } }
@Test public void commitNotification() throws Throwable { setUpHost(false); this.host.stop(); this.host.setPort(0); this.host.start(); this.host.waitForServiceAvailable(ExampleService.FACTORY_LINK); AtomicBoolean isNotified = new AtomicBoolean(false); AtomicLong sequenceNumber = new AtomicLong(); Operation subscribe = Operation.createPost(UriUtils.buildUri(this.host, ServiceUriPaths.CORE_DOCUMENT_INDEX)) .setReferer(this.host.getUri()); this.host.startSubscriptionService(subscribe, notifyOp -> { isNotified.set(true); sequenceNumber.set(notifyOp.getBody(CommitInfo.class).sequenceNumber); notifyOp.complete(); }); assertFalse("callback should not be called before populating data.(nothing to commit)", isNotified.get()); TestRequestSender sender = this.host.getTestRequestSender(); List<Operation> posts = new ArrayList<>(); for (int i = 0; i < 30; i++) { ExampleServiceState state = new ExampleServiceState(); state.name = "foo-" + i; posts.add(Operation.createPost(this.host, ExampleService.FACTORY_LINK).setBody(state)); } sender.sendAndWait(posts, ExampleServiceState.class); this.host.waitFor("commit callback should be called", isNotified::get); assertTrue("Something should be committed", sequenceNumber.get() > -1); }