DataUpdater { public void addCountData( ExpressionExperiment ee, ArrayDesign targetArrayDesign, DoubleMatrix<String, String> countMatrix, DoubleMatrix<String, String> rpkmMatrix, Integer readLength, Boolean isPairedReads, boolean allowMissingSamples ) { if ( countMatrix == null ) throw new IllegalArgumentException( "You must provide count matrix (rpkm is optional)" ); targetArrayDesign = arrayDesignService.thaw( targetArrayDesign ); Collection<ArrayDesign> ads = experimentService.getArrayDesignsUsed( ee ); if ( ads.size() > 1 ) { } ee = experimentService.thawLite( ee ); ee = this.dealWithMissingSamples( ee, countMatrix, allowMissingSamples ); DoubleMatrix<CompositeSequence, BioMaterial> properCountMatrix = this .matchElementsToRowNames( targetArrayDesign, countMatrix ); this.matchBioMaterialsToColNames( ee, countMatrix, properCountMatrix ); assert !properCountMatrix.getColNames().isEmpty(); assert !properCountMatrix.getRowNames().isEmpty(); Collection<QuantitationType> oldQts = ee.getQuantitationTypes(); QuantitationType log2cpmQt = this.makelog2cpmQt(); for ( QuantitationType oldqt : oldQts ) { if ( oldqt.getName().equals( log2cpmQt.getName() ) ) { log2cpmQt = oldqt; break; } } DoubleMatrix1D librarySize = MatrixStats.colSums( countMatrix ); DoubleMatrix<CompositeSequence, BioMaterial> log2cpmMatrix = MatrixStats .convertToLog2Cpm( properCountMatrix, librarySize ); ExpressionDataDoubleMatrix log2cpmEEMatrix = new ExpressionDataDoubleMatrix( ee, log2cpmQt, log2cpmMatrix ); ee = this.replaceData( ee, targetArrayDesign, log2cpmEEMatrix ); QuantitationType countqt = this.makeCountQt(); for ( QuantitationType oldqt : oldQts ) { if ( oldqt.getName().equals( countqt.getName() ) ) { countqt = oldqt; break; } } ExpressionDataDoubleMatrix countEEMatrix = new ExpressionDataDoubleMatrix( ee, countqt, properCountMatrix ); ee = this.addData( ee, targetArrayDesign, countEEMatrix ); this.addTotalCountInformation( ee, countEEMatrix, readLength, isPairedReads ); if ( rpkmMatrix != null ) { DoubleMatrix<CompositeSequence, BioMaterial> properRPKMMatrix = this .matchElementsToRowNames( targetArrayDesign, rpkmMatrix ); this.matchBioMaterialsToColNames( ee, rpkmMatrix, properRPKMMatrix ); assert !properRPKMMatrix.getColNames().isEmpty(); assert !properRPKMMatrix.getRowNames().isEmpty(); QuantitationType rpkmqt = this.makeRPKMQt(); for ( QuantitationType oldqt : oldQts ) { if ( oldqt.getName().equals( rpkmqt.getName() ) ) { rpkmqt = oldqt; break; } } ExpressionDataDoubleMatrix rpkmEEMatrix = new ExpressionDataDoubleMatrix( ee, rpkmqt, properRPKMMatrix ); this.addData( ee, targetArrayDesign, rpkmEEMatrix ); } }  void addAffyDataFromAPTOutput( ExpressionExperiment ee, String pathToAptOutputFile ); void addCountData( ExpressionExperiment ee, ArrayDesign targetArrayDesign,
            DoubleMatrix<String, String> countMatrix, DoubleMatrix<String, String> rpkmMatrix, Integer readLength,
            Boolean isPairedReads, boolean allowMissingSamples ); void log2cpmFromCounts( ExpressionExperiment ee, QuantitationType qt ); @SuppressWarnings("UnusedReturnValue") // Possible external use ExpressionExperiment replaceData( ExpressionExperiment ee, ArrayDesign targetPlatform, QuantitationType qt,
            DoubleMatrix<String, String> data ); void reprocessAffyDataFromCel( ExpressionExperiment ee );  }
@Test public void testLoadRNASeqData() throws Exception { geoService.setGeoDomainObjectGenerator( new GeoDomainObjectGenerator() ); ExpressionExperiment ee; try { Thread.sleep( 1000 ); } catch ( InterruptedException ex ) { Thread.currentThread().interrupt(); } try { Collection<?> results = geoService.fetchAndLoad( "GSE19166", false, false, false ); ee = ( ExpressionExperiment ) results.iterator().next(); } catch ( AlreadyExistsInSystemException e ) { ee = ( ExpressionExperiment ) ( ( List<?> ) e.getData() ).get( 0 ); } ee = experimentService.thaw( ee ); DoubleMatrixReader reader = new DoubleMatrixReader(); DoubleMatrix<String, String> countMatrix; DoubleMatrix<String, String> rpkmMatrix; try (InputStream countData = this.getClass() .getResourceAsStream( "/data/loader/expression/flatfileload/GSE19166_expression_count.test.txt" ); InputStream rpkmData = this.getClass().getResourceAsStream( "/data/loader/expression/flatfileload/GSE19166_expression_RPKM.test.txt" )) { countMatrix = reader.read( countData ); rpkmMatrix = reader.read( rpkmData ); } List<String> probeNames = countMatrix.getRowNames(); assertEquals( 199, probeNames.size() ); targetArrayDesign = this .getTestPersistentArrayDesign( probeNames, taxonService.findByCommonName( "human" ) ); targetArrayDesign = arrayDesignService.thaw( targetArrayDesign ); assertEquals( 199, targetArrayDesign.getCompositeSequences().size() ); dataUpdater.addCountData( ee, targetArrayDesign, countMatrix, rpkmMatrix, 36, true, false ); ee = experimentService.load( ee.getId() ); ee = experimentService.thaw( ee ); assertEquals( 4, ee.getQuantitationTypes().size() ); for ( BioAssay ba : ee.getBioAssays() ) { assertEquals( targetArrayDesign, ba.getArrayDesignUsed() ); } assertNotNull( ee.getNumberOfDataVectors() ); assertEquals( 199, ee.getNumberOfDataVectors().intValue() ); ExpressionDataDoubleMatrix mat = dataMatrixService.getProcessedExpressionDataMatrix( ee ); assertEquals( 199, mat.rows() ); TestUtils.assertBAs( ee, targetArrayDesign, "GSM475204", 3949585 ); assertEquals( 3 * 199, ee.getRawExpressionDataVectors().size() ); assertEquals( 199, ee.getProcessedExpressionDataVectors().size() ); for ( ProcessedExpressionDataVector v : ee.getProcessedExpressionDataVectors() ) { assertNotNull( "Vector rank was not populated (max)", v.getRankByMax() ); assertNotNull( "Vector rank was not populated (mean)", v.getRankByMean() ); } Collection<DoubleVectorValueObject> processedDataArrays = dataVectorService.getProcessedDataArrays( ee ); assertEquals( 199, processedDataArrays.size() ); for ( DoubleVectorValueObject v : processedDataArrays ) { assertEquals( 6, v.getBioAssays().size() ); } assertTrue( !dataVectorService.getProcessedDataVectors( experimentService.load( ee.getId() ) ).isEmpty() ); dataUpdater.addCountData( ee, targetArrayDesign, countMatrix, rpkmMatrix, 36, true, false ); ee = experimentService.load( ee.getId() ); ee = this.experimentService.thawLite( ee ); assertEquals( 4, ee.getQuantitationTypes().size() ); }
@Test public void testLoadRNASeqDataWithMissingSamples() throws Exception { geoService.setGeoDomainObjectGenerator( new GeoDomainObjectGenerator() ); ExpressionExperiment ee = experimentService.findByShortName( "GSE29006" ); if ( ee != null ) { experimentService.remove( ee ); } assertTrue( experimentService.findByShortName( "GSE29006" ) == null ); try { Thread.sleep( 1000 ); } catch ( InterruptedException ex ) { Thread.currentThread().interrupt(); } try { Collection<?> results = geoService.fetchAndLoad( "GSE29006", false, false, false ); ee = ( ExpressionExperiment ) results.iterator().next(); } catch ( AlreadyExistsInSystemException e ) { throw new IllegalStateException( "Need to remove this data set before test is run" ); } ee = experimentService.thaw( ee ); DoubleMatrixReader reader = new DoubleMatrixReader(); try (InputStream countData = this.getClass() .getResourceAsStream( "/data/loader/expression/flatfileload/GSE29006_expression_count.test.txt" ); InputStream rpkmData = this.getClass().getResourceAsStream( "/data/loader/expression/flatfileload/GSE29006_expression_RPKM.test.txt" )) { DoubleMatrix<String, String> countMatrix = reader.read( countData ); DoubleMatrix<String, String> rpkmMatrix = reader.read( rpkmData ); List<String> probeNames = countMatrix.getRowNames(); targetArrayDesign = this .getTestPersistentArrayDesign( probeNames, taxonService.findByCommonName( "human" ) ); targetArrayDesign = arrayDesignService.thaw( targetArrayDesign ); try { dataUpdater.addCountData( ee, targetArrayDesign, countMatrix, rpkmMatrix, 36, true, false ); fail( "Should have gotten an exception" ); } catch ( IllegalArgumentException e ) { } dataUpdater.addCountData( ee, targetArrayDesign, countMatrix, rpkmMatrix, 36, true, true ); } ee = experimentService.thaw( ee ); for ( BioAssay ba : ee.getBioAssays() ) { assertEquals( targetArrayDesign, ba.getArrayDesignUsed() ); } ExpressionDataDoubleMatrix mat = dataMatrixService.getProcessedExpressionDataMatrix( ee ); assertEquals( 199, mat.rows() ); assertTrue( mat.getQuantitationTypes().iterator().next().getName().startsWith( "log2cpm" ) ); assertEquals( 4, ee.getBioAssays().size() ); assertEquals( 199 * 3, ee.getRawExpressionDataVectors().size() ); assertEquals( 199, ee.getProcessedExpressionDataVectors().size() ); Collection<DoubleVectorValueObject> processedDataArrays = dataVectorService.getProcessedDataArrays( ee ); assertEquals( 199, processedDataArrays.size() ); TestUtils.assertBAs( ee, targetArrayDesign, "GSM718709", 320383 ); for ( DoubleVectorValueObject v : processedDataArrays ) { assertEquals( 4, v.getBioAssays().size() ); } }