void (@NotNull Project project, @NotNull CheckCancelOnlyProgressIndicator indicator, @NotNull IndexingProgressReporter progressReporter, @NotNull List<IndexableFilesIterator> providers, @NotNull ProjectScanningHistoryImpl projectScanningHistory) { if (providers.isEmpty()) { return; } ProjectIndexingDependenciesService projectIndexingDependenciesService = myProject.getService(ProjectIndexingDependenciesService.class); final ScanningRequestToken scanningRequest = myOnProjectOpen ? projectIndexingDependenciesService.newScanningTokenOnProjectOpen() : projectIndexingDependenciesService.newScanningToken(); myFutureScanningRequestToken.markSuccessful(); projectIndexingDependenciesService.completeToken(myFutureScanningRequestToken); List<IndexableFileScanner.@NotNull ScanSession> sessions = ContainerUtil.map(IndexableFileScanner.EP_NAME.getExtensionList(), scanner -> scanner.startSession(project)); IndexableFilesDeduplicateFilter indexableFilesDeduplicateFilter = IndexableFilesDeduplicateFilter.create(); progressReporter.setText(IndexingBundle.message("progress.indexing.scanning")); progressReporter.setSubTasksCount(providers.size()); // Workaround for concurrent modification of the [scanningHistory]. // PushedFilePropertiesUpdaterImpl.invokeConcurrentlyIfPossible may finish earlier than some of its spawned tasks. // And some scanning statistics may be tried to be added to the [scanningHistory], // leading to ConcurrentModificationException in the statistics' processor. Ref<Boolean> allTasksFinished = Ref.create(false); IndexingReasonExplanationLogger sharedExplanationLogger = new IndexingReasonExplanationLogger(); List<Runnable> tasks = ContainerUtil.map(providers, provider -> { ScanningStatistics scanningStatistics = new ScanningStatistics(provider.getDebugName()); scanningStatistics.setProviderRoots(provider, project); IndexableSetOrigin origin = provider.getOrigin(); List<IndexableFileScanner.@NotNull IndexableFileVisitor> fileScannerVisitors = ContainerUtil.mapNotNull(sessions, s -> s.createVisitor(origin)); IndexableFilesDeduplicateFilter thisProviderDeduplicateFilter = IndexableFilesDeduplicateFilter.createDelegatingTo(indexableFilesDeduplicateFilter); ProgressManager.checkCanceled(); // give a chance to suspend indexing return () -> { long providerScanningStartTime = System.nanoTime(); try ( PerProviderSink perProviderSink = project.getService(PerProjectIndexingQueue.class) .getSink(provider, projectScanningHistory.getScanningSessionId()); IndexingSubTaskProgressReporter subTaskReporter = progressReporter.getSubTaskReporter() ) { subTaskReporter.setText(provider.getRootsScanningProgressText()); List<Pair<VirtualFile, List<VirtualFile>>> rootsAndFiles = new ArrayList<>(); Function<@Nullable VirtualFile, ContentIterator> singleProviderIteratorFactory = root -> { List<VirtualFile> files = new ArrayList<>(1024); rootsAndFiles.add(new Pair<>(root, files)); return fileOrDir -> { // we apply scanners here, because scanners may mark directory as excluded, and we should skip excluded subtrees // (e.g., JSDetectingProjectFileScanner.startSession will exclude "node_modules" directories during scanning) PushedFilePropertiesUpdaterImpl.applyScannersToFile(fileOrDir, fileScannerVisitors); return files.add(fileOrDir); }; }; scanningStatistics.startVfsIterationAndScanningApplication(); provider.iterateFilesInRoots(project, singleProviderIteratorFactory, thisProviderDeduplicateFilter); scanningStatistics.tryFinishVfsIterationAndScanningApplication(); scanningStatistics.startFileChecking(); for (Pair<VirtualFile, List<VirtualFile>> rootAndFiles : rootsAndFiles) { UnindexedFilesFinder finder = new UnindexedFilesFinder(project, sharedExplanationLogger, myIndex, getForceReindexingTrigger(), rootAndFiles.getFirst(), scanningRequest); var rootIterator = new SingleProviderIterator(project, indicator, provider, finder, scanningStatistics, perProviderSink); if (!rootIterator.mayBeUsed()) { LOG.warn("Iterator based on " + provider + " can't be used."); continue; } rootAndFiles.getSecond().forEach(it -> rootIterator.processFile(it)); } scanningStatistics.tryFinishFilesChecking(); perProviderSink.commit(); } catch (ProcessCanceledException pce) { scanningRequest.markUnsuccessful(); throw pce; } catch (Exception e) { scanningRequest.markUnsuccessful(); // CollectingIterator should skip failing files by itself. But if provider.iterateFiles cannot iterate files and throws exception, // we want to ignore the whole origin and let other origins complete normally. LOG.error("Error while scanning files of " + provider.getDebugName() + "\n" + "To reindex files under this origin IDEA has to be restarted", e); } finally { scanningStatistics.tryFinishVfsIterationAndScanningApplication(); scanningStatistics.tryFinishFilesChecking(); scanningStatistics.setTotalOneThreadTimeWithPauses(System.nanoTime() - providerScanningStartTime); scanningStatistics.setNumberOfSkippedFiles(thisProviderDeduplicateFilter.getNumberOfSkippedFiles()); synchronized (allTasksFinished) { if (!allTasksFinished.get()) { projectScanningHistory.addScanningStatistics(scanningStatistics); } } } }; }); LOG.info("Scanning of " + myProject.getName() + " uses " + UnindexedFilesUpdater.getNumberOfScanningThreads() + " scanning threads"); try { PushedFilePropertiesUpdaterImpl.invokeConcurrentlyIfPossible(tasks); } finally { synchronized (allTasksFinished) { allTasksFinished.set(true); projectIndexingDependenciesService.completeToken(scanningRequest); } } }