@Test @Ignore @Transactional(propagation = Propagation.REQUIRES_NEW) public void performanceTest() throws Exception { Reference hg38 = EntityHelper.createG38Reference(referenceGenomeManager.createReferenceId()); referenceGenomeManager.create(hg38); FeatureIndexedFileRegistrationRequest request = new FeatureIndexedFileRegistrationRequest(); request.setReferenceId(hg38.getId()); request.setPath("/home/kite/Documents/sampleData/Dream.set3.VarDict.SV.vcf"); VcfFile vcfFile1 = vcfManager.registerVcfFile(request); request.setPath("/home/kite/Documents/sampleData/synthetic.challenge.set3.tumor.20pctmasked.truth.vcf"); VcfFile vcfFile2 = vcfManager.registerVcfFile(request); Project project = new Project(); project.setName(TEST_PROJECT_NAME + 1); project.setItems(Arrays.asList(new ProjectItem(new BiologicalDataItem(vcfFile1.getBioDataItemId())), new ProjectItem(new BiologicalDataItem(vcfFile2.getBioDataItemId())))); projectManager.create(project); IndexSearchResult<VcfIndexEntry> entriesRes = featureIndexManager.filterVariations(new VcfFilterForm(), project.getId()); List<VcfIndexEntry> entries = entriesRes.getEntries(); Assert.assertFalse(entries.isEmpty()); logger.info("!! Variations count: {}", entries.size()); TestUtils.warmUp(() -> featureIndexManager.filterVariations(new VcfFilterForm(), project.getId()), PERFORMANCE_TEST_WARMING_COUNT); double averageTime = TestUtils.measurePerformance( () -> featureIndexManager.filterVariations(new VcfFilterForm(), project.getId()), PERFORMANCE_TEST_ATTEMPTS_COUNT); logger.info("!! Performing index search took: {} ms", averageTime); TestUtils.warmUp(() -> featureIndexManager.filterVariations(new VcfFilterForm(), project.getId()), PERFORMANCE_TEST_WARMING_COUNT); averageTime = TestUtils.measurePerformance( () -> featureIndexManager.filterVariations(new VcfFilterForm(), project.getId()), PERFORMANCE_TEST_ATTEMPTS_COUNT); logger.info("!! Performing index search paging took: {} ms", averageTime); final VcfFilterForm filterForm = new VcfFilterForm(); filterForm.setPage(1); filterForm.setPageSize(PERFORMANCE_TEST_PAGE_SIZE); TestUtils.warmUp(() -> featureIndexManager.filterVariations(filterForm, project.getId()), PERFORMANCE_TEST_WARMING_COUNT); averageTime = TestUtils.measurePerformance( () -> featureIndexManager.filterVariations(filterForm, project.getId()), PERFORMANCE_TEST_ATTEMPTS_COUNT); logger.info("!! Performing index search single page took: {} ms", averageTime); TestUtils.warmUp( () -> ThreadLocalRandom.current().nextInt(1, entries.size() / PERFORMANCE_TEST_PAGE_SIZE + 1), (page) -> { filterForm.setPage(page); featureIndexManager.filterVariations(filterForm, project.getId()); }, PERFORMANCE_TEST_WARMING_COUNT); List<Double> timings = TestUtils.measurePerformanceTimings( () -> ThreadLocalRandom.current().nextInt(1, entries.size() / PERFORMANCE_TEST_PAGE_SIZE + 1), (page) -> { filterForm.setPage(page); featureIndexManager.filterVariations(filterForm, project.getId()); }, PERFORMANCE_TEST_ATTEMPTS_COUNT); timings.forEach(t -> logger.info("!! Performed index search random page took: {}", t)); averageTime = TestUtils.calculateAverage(timings); logger.info("!! Average Performing index search random page took: {} ms", averageTime); }
public IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm, long projectId) throws IOException { Project project = projectManager.load(projectId); List<VcfFile> files = project.getItems().stream() .filter(i -> i.getBioDataItem().getFormat() == BiologicalDataItemFormat.VCF) .map(i -> (VcfFile) i.getBioDataItem()) .collect(Collectors.toList()); return getVcfSearchResult(filterForm, files); }
FeatureIndexManager { public IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm, long projectId) throws IOException { Project project = projectManager.load(projectId); List<VcfFile> files = project.getItems().stream() .filter(i -> i.getBioDataItem().getFormat() == BiologicalDataItemFormat.VCF) .map(i -> (VcfFile) i.getBioDataItem()) .collect(Collectors.toList()); return getVcfSearchResult(filterForm, files); } }
FeatureIndexManager { public IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm, long projectId) throws IOException { Project project = projectManager.load(projectId); List<VcfFile> files = project.getItems().stream() .filter(i -> i.getBioDataItem().getFormat() == BiologicalDataItemFormat.VCF) .map(i -> (VcfFile) i.getBioDataItem()) .collect(Collectors.toList()); return getVcfSearchResult(filterForm, files); }  }
FeatureIndexManager { public IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm, long projectId) throws IOException { Project project = projectManager.load(projectId); List<VcfFile> files = project.getItems().stream() .filter(i -> i.getBioDataItem().getFormat() == BiologicalDataItemFormat.VCF) .map(i -> (VcfFile) i.getBioDataItem()) .collect(Collectors.toList()); return getVcfSearchResult(filterForm, files); }  void deleteFromIndexByFileId(final long projectId, List<Pair<FeatureType, Long>> fileIds); Set<String> searchGenesInVcfFilesInProject(long projectId, String gene, List<Long> vcfFileIds); Set<String> searchGenesInVcfFiles(String gene, List<Long> vcfFileIds); List<Chromosome> filterChromosomes(VcfFilterForm filterForm, long projectId); List<Chromosome> filterChromosomes(VcfFilterForm filterForm); IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm, long projectId); int getTotalPagesCount(VcfFilterForm filterForm); int getTotalPagesCount(VcfFilterForm filterForm, long projectId); List<Group> groupVariations(VcfFilterForm filterForm, long projectId, String groupByField); List<Group> groupVariations(VcfFilterForm filterForm, String groupByField); IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm); IndexSearchResult searchFeaturesInProject(String featureId, long projectId); IndexSearchResult searchFeaturesByReference(String featureId, long referenceId); VcfFilterInfo loadVcfFilterInfoForProject(long projectId); void buildIndexForFile(FeatureFile featureFile); Set<String> fetchGeneIds(int start, int end, List<GeneFile> geneFiles, Chromosome chromosome); void writeLuceneIndexForFile(final FeatureFile featureFile,
            final List<? extends FeatureIndexEntry> entries,
            VcfFilterInfo vcfFilterInfo); void makeIndexForBedReader(BedFile bedFile, AbstractFeatureReader<NggbBedFeature, LineIterator> reader,
                                      Map<String, Chromosome> chromosomeMap); void makeIndexForVcfReader(VcfFile vcfFile, FeatureReader<VariantContext> reader, List<GeneFile> geneFiles,
                                       Map<String, Chromosome> chromosomeMap, VcfFilterInfo info); void processGeneFile(GeneFile geneFile, final Map<String, Chromosome> chromosomeMap,
            boolean full); void addGeneFeatureToIndex(List<FeatureIndexEntry> allEntries, GeneFeature feature,
                                     Map<String, Chromosome> chromosomeMap); }
FeatureIndexManager { public IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm, long projectId) throws IOException { Project project = projectManager.load(projectId); List<VcfFile> files = project.getItems().stream() .filter(i -> i.getBioDataItem().getFormat() == BiologicalDataItemFormat.VCF) .map(i -> (VcfFile) i.getBioDataItem()) .collect(Collectors.toList()); return getVcfSearchResult(filterForm, files); }  void deleteFromIndexByFileId(final long projectId, List<Pair<FeatureType, Long>> fileIds); Set<String> searchGenesInVcfFilesInProject(long projectId, String gene, List<Long> vcfFileIds); Set<String> searchGenesInVcfFiles(String gene, List<Long> vcfFileIds); List<Chromosome> filterChromosomes(VcfFilterForm filterForm, long projectId); List<Chromosome> filterChromosomes(VcfFilterForm filterForm); IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm, long projectId); int getTotalPagesCount(VcfFilterForm filterForm); int getTotalPagesCount(VcfFilterForm filterForm, long projectId); List<Group> groupVariations(VcfFilterForm filterForm, long projectId, String groupByField); List<Group> groupVariations(VcfFilterForm filterForm, String groupByField); IndexSearchResult<VcfIndexEntry> filterVariations(VcfFilterForm filterForm); IndexSearchResult searchFeaturesInProject(String featureId, long projectId); IndexSearchResult searchFeaturesByReference(String featureId, long referenceId); VcfFilterInfo loadVcfFilterInfoForProject(long projectId); void buildIndexForFile(FeatureFile featureFile); Set<String> fetchGeneIds(int start, int end, List<GeneFile> geneFiles, Chromosome chromosome); void writeLuceneIndexForFile(final FeatureFile featureFile,
            final List<? extends FeatureIndexEntry> entries,
            VcfFilterInfo vcfFilterInfo); void makeIndexForBedReader(BedFile bedFile, AbstractFeatureReader<NggbBedFeature, LineIterator> reader,
                                      Map<String, Chromosome> chromosomeMap); void makeIndexForVcfReader(VcfFile vcfFile, FeatureReader<VariantContext> reader, List<GeneFile> geneFiles,
                                       Map<String, Chromosome> chromosomeMap, VcfFilterInfo info); void processGeneFile(GeneFile geneFile, final Map<String, Chromosome> chromosomeMap,
            boolean full); void addGeneFeatureToIndex(List<FeatureIndexEntry> allEntries, GeneFeature feature,
                                     Map<String, Chromosome> chromosomeMap);  }