@Test public void accessODLAfterRemovedByMemoryPressure() throws Throwable { LuceneDocumentIndexService indexService = new LuceneDocumentIndexService(); this.host = VerificationHost.create(0); this.host.setDocumentIndexingService(indexService); this.host.start(); this.host.startFactory(new ExampleODLService()); this.host.waitForServiceAvailable(ExampleODLService.FACTORY_LINK); TestRequestSender sender = this.host.getTestRequestSender(); List<Operation> posts = new ArrayList<>(); for (int i = 0; i < 30; i++) { ExampleServiceState state = new ExampleServiceState(); state.name = "foo-" + i; state.documentSelfLink = state.name; posts.add(Operation.createPost(this.host, ExampleODLService.FACTORY_LINK).setBody(state)); } List<ExampleServiceState> states = sender.sendAndWait(posts, ExampleServiceState.class); List<Operation> deletes = states.stream().map(state -> Operation.createDelete(this.host, state.documentSelfLink) .toggleOption(OperationOption.FORWARDING_DISABLED, true) .toggleOption(OperationOption.INDEXING_DISABLED, true) ).collect(toList()); sender.sendAndWait(deletes); indexService.updateMapMemoryLimit = 1; indexService.applyMemoryLimitToDocumentUpdateInfo(); List<Operation> gets = states.stream() .map(state -> Operation.createGet(this.host, state.documentSelfLink)) .collect(toList()); sender.sendAndWait(gets); }
void applyMemoryLimitToDocumentUpdateInfo() { long memThresholdBytes = this.updateMapMemoryLimit; final int bytesPerLinkEstimate = 64; int count = 0; if (allowStats()) { setStat(STAT_NAME_VERSION_CACHE_ENTRY_COUNT, this.updatesPerLink.size()); } synchronized (this.searchSync) { if (this.updatesPerLink.isEmpty()) { return; } if (memThresholdBytes > this.updatesPerLink.size() * bytesPerLinkEstimate) { return; } Iterator<Entry<String, DocumentUpdateInfo>> li = this.updatesPerLink.entrySet() .iterator(); while (li.hasNext()) { Entry<String, DocumentUpdateInfo> e = li.next(); if (getHost().getServiceStage(e.getKey()) == null) { count++; li.remove(); } } this.writerUpdateTimeMicros = Utils.getNowMicrosUtc(); } if (count == 0) { return; } this.serviceRemovalDetectedTimeMicros = Utils.getNowMicrosUtc(); logInfo("Cleared %d document update entries", count); }
LuceneDocumentIndexService extends StatelessService { void applyMemoryLimitToDocumentUpdateInfo() { long memThresholdBytes = this.updateMapMemoryLimit; final int bytesPerLinkEstimate = 64; int count = 0; if (allowStats()) { setStat(STAT_NAME_VERSION_CACHE_ENTRY_COUNT, this.updatesPerLink.size()); } synchronized (this.searchSync) { if (this.updatesPerLink.isEmpty()) { return; } if (memThresholdBytes > this.updatesPerLink.size() * bytesPerLinkEstimate) { return; } Iterator<Entry<String, DocumentUpdateInfo>> li = this.updatesPerLink.entrySet() .iterator(); while (li.hasNext()) { Entry<String, DocumentUpdateInfo> e = li.next(); if (getHost().getServiceStage(e.getKey()) == null) { count++; li.remove(); } } this.writerUpdateTimeMicros = Utils.getNowMicrosUtc(); } if (count == 0) { return; } this.serviceRemovalDetectedTimeMicros = Utils.getNowMicrosUtc(); logInfo("Cleared %d document update entries", count); } }
LuceneDocumentIndexService extends StatelessService { void applyMemoryLimitToDocumentUpdateInfo() { long memThresholdBytes = this.updateMapMemoryLimit; final int bytesPerLinkEstimate = 64; int count = 0; if (allowStats()) { setStat(STAT_NAME_VERSION_CACHE_ENTRY_COUNT, this.updatesPerLink.size()); } synchronized (this.searchSync) { if (this.updatesPerLink.isEmpty()) { return; } if (memThresholdBytes > this.updatesPerLink.size() * bytesPerLinkEstimate) { return; } Iterator<Entry<String, DocumentUpdateInfo>> li = this.updatesPerLink.entrySet() .iterator(); while (li.hasNext()) { Entry<String, DocumentUpdateInfo> e = li.next(); if (getHost().getServiceStage(e.getKey()) == null) { count++; li.remove(); } } this.writerUpdateTimeMicros = Utils.getNowMicrosUtc(); } if (count == 0) { return; } this.serviceRemovalDetectedTimeMicros = Utils.getNowMicrosUtc(); logInfo("Cleared %d document update entries", count); } LuceneDocumentIndexService();  LuceneDocumentIndexService(String indexDirectory); }
LuceneDocumentIndexService extends StatelessService { void applyMemoryLimitToDocumentUpdateInfo() { long memThresholdBytes = this.updateMapMemoryLimit; final int bytesPerLinkEstimate = 64; int count = 0; if (allowStats()) { setStat(STAT_NAME_VERSION_CACHE_ENTRY_COUNT, this.updatesPerLink.size()); } synchronized (this.searchSync) { if (this.updatesPerLink.isEmpty()) { return; } if (memThresholdBytes > this.updatesPerLink.size() * bytesPerLinkEstimate) { return; } Iterator<Entry<String, DocumentUpdateInfo>> li = this.updatesPerLink.entrySet() .iterator(); while (li.hasNext()) { Entry<String, DocumentUpdateInfo> e = li.next(); if (getHost().getServiceStage(e.getKey()) == null) { count++; li.remove(); } } this.writerUpdateTimeMicros = Utils.getNowMicrosUtc(); } if (count == 0) { return; } this.serviceRemovalDetectedTimeMicros = Utils.getNowMicrosUtc(); logInfo("Cleared %d document update entries", count); } LuceneDocumentIndexService();  LuceneDocumentIndexService(String indexDirectory); static void setImplicitQueryResultLimit(int limit); static int getImplicitQueryResultLimit(); static void setImplicitQueryProcessingPageSize(int limit); static int getImplicitQueryProcessingPageSize(); static void setIndexFileCountThresholdForWriterRefresh(int count); static int getIndexFileCountThresholdForWriterRefresh(); static void setExpiredDocumentSearchThreshold(int count); static int getExpiredDocumentSearchThreshold(); static void setVersionRetentionBulkCleanupThreshold(int count); static int getVersionRetentionBulkCleanupThreshold(); static void setVersionRetentionServiceThreshold(int count); static int getVersionRetentionServiceThreshold(); static long getSearcherRefreshIntervalMicros(); static void setSearcherRefreshIntervalMicros(long interval); static void setMetadataUpdateMaxQueueDepth(int depth); static int getMetadataUpdateMaxQueueDepth(); @Override void handleStart(final Operation post); IndexWriter createWriter(File directory, boolean doUpgrade); @Override void authorizeRequest(Operation op); @Override void handleRequest(Operation op); void handleGetImpl(Operation get); void handleDeleteImpl(Operation delete); @Override URI getUri(); @Override void handleMaintenance(Operation post); }
LuceneDocumentIndexService extends StatelessService { void applyMemoryLimitToDocumentUpdateInfo() { long memThresholdBytes = this.updateMapMemoryLimit; final int bytesPerLinkEstimate = 64; int count = 0; if (allowStats()) { setStat(STAT_NAME_VERSION_CACHE_ENTRY_COUNT, this.updatesPerLink.size()); } synchronized (this.searchSync) { if (this.updatesPerLink.isEmpty()) { return; } if (memThresholdBytes > this.updatesPerLink.size() * bytesPerLinkEstimate) { return; } Iterator<Entry<String, DocumentUpdateInfo>> li = this.updatesPerLink.entrySet() .iterator(); while (li.hasNext()) { Entry<String, DocumentUpdateInfo> e = li.next(); if (getHost().getServiceStage(e.getKey()) == null) { count++; li.remove(); } } this.writerUpdateTimeMicros = Utils.getNowMicrosUtc(); } if (count == 0) { return; } this.serviceRemovalDetectedTimeMicros = Utils.getNowMicrosUtc(); logInfo("Cleared %d document update entries", count); } LuceneDocumentIndexService();  LuceneDocumentIndexService(String indexDirectory); static void setImplicitQueryResultLimit(int limit); static int getImplicitQueryResultLimit(); static void setImplicitQueryProcessingPageSize(int limit); static int getImplicitQueryProcessingPageSize(); static void setIndexFileCountThresholdForWriterRefresh(int count); static int getIndexFileCountThresholdForWriterRefresh(); static void setExpiredDocumentSearchThreshold(int count); static int getExpiredDocumentSearchThreshold(); static void setVersionRetentionBulkCleanupThreshold(int count); static int getVersionRetentionBulkCleanupThreshold(); static void setVersionRetentionServiceThreshold(int count); static int getVersionRetentionServiceThreshold(); static long getSearcherRefreshIntervalMicros(); static void setSearcherRefreshIntervalMicros(long interval); static void setMetadataUpdateMaxQueueDepth(int depth); static int getMetadataUpdateMaxQueueDepth(); @Override void handleStart(final Operation post); IndexWriter createWriter(File directory, boolean doUpgrade); @Override void authorizeRequest(Operation op); @Override void handleRequest(Operation op); void handleGetImpl(Operation get); void handleDeleteImpl(Operation delete); @Override URI getUri(); @Override void handleMaintenance(Operation post); static final String SELF_LINK; static final int QUERY_THREAD_COUNT; static final int UPDATE_THREAD_COUNT; static final int QUERY_QUEUE_DEPTH; static final int UPDATE_QUEUE_DEPTH; static final boolean QUERY_LOGGING; static final boolean UPDATE_LOGGING; static final String FILE_PATH_LUCENE; static final int DEFAULT_INDEX_FILE_COUNT_THRESHOLD_FOR_WRITER_REFRESH; static final int DEFAULT_INDEX_SEARCHER_COUNT_THRESHOLD; static final int MIN_QUERY_RESULT_LIMIT; static final int DEFAULT_QUERY_RESULT_LIMIT; static final int DEFAULT_QUERY_PAGE_RESULT_LIMIT; static final int DEFAULT_EXPIRED_DOCUMENT_SEARCH_THRESHOLD; static final int DEFAULT_METADATA_UPDATE_MAX_QUEUE_DEPTH; static final long DEFAULT_PAGINATED_SEARCHER_EXPIRATION_DELAY; static final String PROPERTY_NAME_QUERY_QUEUE_DEPTH; static final String PROPERTY_NAME_UPDATE_QUEUE_DEPTH; static final String STAT_NAME_ACTIVE_QUERY_FILTERS; static final String STAT_NAME_ACTIVE_PAGINATED_QUERIES; static final String STAT_NAME_COMMIT_COUNT; static final String STAT_NAME_INDEX_LOAD_RETRY_COUNT; static final String STAT_NAME_COMMIT_DURATION_MICROS; static final String STAT_NAME_GROUP_QUERY_COUNT; static final String STAT_NAME_QUERY_DURATION_MICROS; static final String STAT_NAME_GROUP_QUERY_DURATION_MICROS; static final String STAT_NAME_QUERY_SINGLE_DURATION_MICROS; static final String STAT_NAME_QUERY_ALL_VERSIONS_DURATION_MICROS; static final String STAT_NAME_RESULT_PROCESSING_DURATION_MICROS; static final String STAT_NAME_INDEXED_FIELD_COUNT; static final String STAT_NAME_INDEXED_DOCUMENT_COUNT; static final String STAT_NAME_FORCED_UPDATE_DOCUMENT_DELETE_COUNT; static final String STAT_NAME_FIELD_COUNT_PER_DOCUMENT; static final String STAT_NAME_INDEXING_DURATION_MICROS; static final String STAT_NAME_SEARCHER_UPDATE_COUNT; static final String STAT_NAME_SEARCHER_REUSE_BY_DOCUMENT_KIND_COUNT; static final String STAT_NAME_PAGINATED_SEARCHER_UPDATE_COUNT; static final String STAT_NAME_PAGINATED_SEARCHER_FORCE_DELETION_COUNT; static final String STAT_NAME_WRITER_ALREADY_CLOSED_EXCEPTION_COUNT; static final String STAT_NAME_READER_ALREADY_CLOSED_EXCEPTION_COUNT; static final String STAT_NAME_SERVICE_DELETE_COUNT; static final String STAT_NAME_DOCUMENT_EXPIRATION_COUNT; static final String STAT_NAME_DOCUMENT_EXPIRATION_FORCED_MAINTENANCE_COUNT; static final String STAT_NAME_METADATA_INDEXING_UPDATE_COUNT; static final String STAT_NAME_VERSION_CACHE_LOOKUP_COUNT; static final String STAT_NAME_VERSION_CACHE_MISS_COUNT; static final String STAT_NAME_VERSION_CACHE_ENTRY_COUNT; static final String STAT_NAME_MAINTENANCE_SEARCHER_REFRESH_DURATION_MICROS; static final String STAT_NAME_MAINTENANCE_DOCUMENT_EXPIRATION_DURATION_MICROS; static final String STAT_NAME_MAINTENANCE_VERSION_RETENTION_DURATION_MICROS; static final String STAT_NAME_MAINTENANCE_METADATA_INDEXING_DURATION_MICROS; static final String STAT_NAME_DOCUMENT_KIND_QUERY_COUNT_FORMAT; static final String STAT_NAME_NON_DOCUMENT_KIND_QUERY_COUNT; static final String STAT_NAME_SINGLE_QUERY_BY_FACTORY_COUNT_FORMAT; static final String STAT_NAME_PREFIX_UPDATE_QUEUE_DEPTH; static final String STAT_NAME_FORMAT_UPDATE_QUEUE_DEPTH; static final String STAT_NAME_PREFIX_QUERY_QUEUE_DEPTH; static final String STAT_NAME_FORMAT_QUERY_QUEUE_DEPTH; }