StatisticsOp extends Operator { @Override public void initialize() throws OperatorException { setDummyTargetProduct(); validateInput(); final StatisticComputer statisticComputer = new StatisticComputer(shapefile, bandConfigurations, Util.computeBinCount(accuracy), getLogger()); final ProductValidator productValidator = new ProductValidator(Arrays.asList(bandConfigurations), startDate, endDate, getLogger()); final ProductLoop productLoop = new ProductLoop(new ProductLoader(), productValidator, statisticComputer, getLogger()); productLoop.loop(sourceProducts, getProductsToLoad()); if (startDate == null) { startDate = productLoop.getOldestDate(); } if (endDate == null) { endDate = productLoop.getNewestDate(); } final String[] productNames = productLoop.getProductNames(); if (productNames.length == 0) { throw new OperatorException("No input products found that matches the criteria."); } final Map<BandConfiguration, StatisticComputer.StxOpMapping> results = statisticComputer.getResults(); final String[] algorithmNames = getAlgorithmNames(percentiles); final List<String> bandNamesList = new ArrayList<String>(); for (BandConfiguration bandConfiguration : bandConfigurations) { if (bandConfiguration.sourceBandName != null) { bandNamesList.add(bandConfiguration.sourceBandName); } else { bandNamesList.add(bandConfiguration.expression.replace(" ", "_")); } } final String[] bandNames = bandNamesList.toArray(new String[bandNamesList.size()]); regionNames.clear(); for (StatisticComputer.StxOpMapping stxOpMapping : results.values()) { regionNames.addAll(stxOpMapping.summaryMap.keySet()); } if (regionNames.size() == 0) { getLogger().warning("No statistics computed because no input product intersects any feature from the given shapefile."); return; } final StatisticsOutputContext statisticsOutputContext = StatisticsOutputContext.create(productNames, bandNames, algorithmNames, startDate, endDate, regionNames.toArray(new String[regionNames.size()])); setupOutputter(); for (StatisticsOutputter statisticsOutputter : statisticsOutputters) { statisticsOutputter.initialiseOutput(statisticsOutputContext); } for (Map.Entry<BandConfiguration, StatisticComputer.StxOpMapping> bandConfigurationStxOpMappingEntry : results.entrySet()) { final BandConfiguration bandConfiguration = bandConfigurationStxOpMappingEntry.getKey(); final String bandName; if (bandConfiguration.sourceBandName != null) { bandName = bandConfiguration.sourceBandName; } else { bandName = bandConfiguration.expression.replace(" ", "_"); } final StatisticComputer.StxOpMapping stxOpMapping = bandConfigurationStxOpMappingEntry.getValue(); final Map<String, SummaryStxOp> summaryMap = stxOpMapping.summaryMap; final Map<String, HistogramStxOp> histogramMap = stxOpMapping.histogramMap; for (String regionName : summaryMap.keySet()) { final SummaryStxOp summaryStxOp = summaryMap.get(regionName); final Histogram histogram = histogramMap.get(regionName).getHistogram(); final HashMap<String, Number> stxMap = new HashMap<String, Number>(); if (histogram.getTotals()[0] == 0) { stxMap.put(MINIMUM, FILL_VALUE); stxMap.put(MAXIMUM, FILL_VALUE); stxMap.put(AVERAGE, FILL_VALUE); stxMap.put(SIGMA, FILL_VALUE); stxMap.put(TOTAL, 0); stxMap.put(MEDIAN, FILL_VALUE); for (int percentile : percentiles) { stxMap.put(getPercentileName(percentile), FILL_VALUE); } } else { stxMap.put(MINIMUM, summaryStxOp.getMinimum()); stxMap.put(MAXIMUM, summaryStxOp.getMaximum()); stxMap.put(AVERAGE, summaryStxOp.getMean()); stxMap.put(SIGMA, summaryStxOp.getStandardDeviation()); stxMap.put(TOTAL, histogram.getTotals()[0]); stxMap.put(MEDIAN, histogram.getPTileThreshold(0.5)[0]); for (int percentile : percentiles) { stxMap.put(getPercentileName(percentile), computePercentile(percentile, histogram)); } } stxMap.put(MAX_ERROR, Util.getBinWidth(histogram)); for (StatisticsOutputter statisticsOutputter : statisticsOutputters) { statisticsOutputter.addToOutput(bandName, regionName, stxMap); } } } try { for (StatisticsOutputter statisticsOutputter : statisticsOutputters) { statisticsOutputter.finaliseOutput(); } } catch (IOException e) { throw new OperatorException("Unable to write output.", e); } finally { if (metadataOutputStream != null) { metadataOutputStream.close(); } if (csvOutputStream != null) { csvOutputStream.close(); } if (bandMappingOutputStream != null) { bandMappingOutputStream.close(); } } getLogger().log(Level.INFO, "Successfully computed statistics."); }  @Override void initialize(); static String[] getAlgorithmNames(int[] percentiles); static final String DATETIME_PATTERN; static final String MAXIMUM; static final String MINIMUM; static final String MEDIAN; static final String AVERAGE; static final String SIGMA; static final String MAX_ERROR; static final String TOTAL; static final String PERCENTILE_PREFIX; static final String PERCENTILE_SUFFIX; static final String DEFAULT_PERCENTILES; static final int[] DEFAULT_PERCENTILES_INTS; }
@Test public void testStatisticsOp() throws Exception { final StatisticsOp statisticsOp = createStatisticsOp(); final BandConfiguration bandConfiguration = new BandConfiguration(); bandConfiguration.sourceBandName = "algal_2"; statisticsOp.bandConfigurations = new BandConfiguration[]{bandConfiguration}; statisticsOp.sourceProducts = new Product[]{TestUtil.getTestProduct()}; statisticsOp.shapefile = new File(getClass().getResource("4_pixels.shp").getFile()); final MyOutputter outputter = new MyOutputter(); statisticsOp.statisticsOutputters.add(outputter); statisticsOp.initialize(); assertEquals("4_pixels.1", outputter.region); assertEquals("algal_2", outputter.bandName); assertEquals(4, outputter.pixels); assertEquals(0.804474, outputter.maximum, 1E-3); assertEquals(0.695857, outputter.minimum, 1E-3); assertEquals(0.749427, outputter.average, 1E-3); assertEquals(0.721552, outputter.median, 1E-3); assertEquals(0.049577, outputter.sigma, 1E-3); assertEquals(2, outputter.percentiles.length); assertEquals(0.804474, outputter.percentiles[0], 1E-3); assertEquals(0.804474, outputter.percentiles[1], 1E-3); }
@Test public void testStatisticsOp_WithPrecisePercentiles() throws Exception { final StatisticsOp statisticsOp = createStatisticsOp(); final BandConfiguration bandConfiguration = new BandConfiguration(); bandConfiguration.sourceBandName = "algal_2"; statisticsOp.bandConfigurations = new BandConfiguration[]{bandConfiguration}; statisticsOp.sourceProducts = new Product[]{TestUtil.getTestProduct()}; statisticsOp.shapefile = new File(getClass().getResource("4_pixels.shp").getFile()); statisticsOp.accuracy = 6; final MyOutputter outputter = new MyOutputter(); statisticsOp.statisticsOutputters.add(outputter); statisticsOp.initialize(); assertEquals("4_pixels.1", outputter.region); assertEquals("algal_2", outputter.bandName); assertEquals(4, outputter.pixels); assertEquals(0.804474, outputter.maximum, 1E-6); assertEquals(0.695857, outputter.minimum, 1E-6); assertEquals(0.749427, outputter.average, 1E-6); assertEquals(0.721552, outputter.median, 1E-6); assertEquals(0.049577, outputter.sigma, 1E-6); assertEquals(2, outputter.percentiles.length); assertEquals(0.80447364, outputter.percentiles[0], 1E-6); assertEquals(0.80447364, outputter.percentiles[1], 1E-6); }
@Test public void testStatisticsOp_WithExpression() throws Exception { final StatisticsOp statisticsOp = createStatisticsOp(); final BandConfiguration bandConfiguration = new BandConfiguration(); bandConfiguration.expression = "algal_2 * PI"; statisticsOp.bandConfigurations = new BandConfiguration[]{bandConfiguration}; statisticsOp.sourceProducts = new Product[]{TestUtil.getTestProduct()}; statisticsOp.shapefile = new File(getClass().getResource("4_pixels.shp").getFile()); final MyOutputter outputter = new MyOutputter(); statisticsOp.statisticsOutputters.add(outputter); statisticsOp.initialize(); assertEquals("4_pixels.1", outputter.region); assertEquals("algal_2_*_PI", outputter.bandName); assertEquals(4, outputter.pixels); assertEquals(2.527328, outputter.maximum, 1E-3); assertEquals(2.186098, outputter.minimum, 1E-3); assertEquals(2.354394, outputter.average, 1E-3); assertEquals(2.266823, outputter.median, 1E-3); assertEquals(0.155752, outputter.sigma, 1E-3); assertEquals(2, outputter.percentiles.length); assertEquals(2.527328, outputter.percentiles[0], 1E-3); assertEquals(2.527328, outputter.percentiles[1], 1E-3); }
@Test public void testStatisticsOp_WithValidExpression() throws Exception { final StatisticsOp statisticsOp = createStatisticsOp(); final BandConfiguration bandConfiguration = new BandConfiguration(); bandConfiguration.sourceBandName = "algal_2"; bandConfiguration.validPixelExpression = "algal_2 > 0.7"; statisticsOp.bandConfigurations = new BandConfiguration[]{bandConfiguration}; statisticsOp.sourceProducts = new Product[]{TestUtil.getTestProduct()}; statisticsOp.shapefile = new File(getClass().getResource("4_pixels.shp").getFile()); final MyOutputter outputter = new MyOutputter(); statisticsOp.statisticsOutputters.add(outputter); statisticsOp.initialize(); assertEquals("4_pixels.1", outputter.region); assertEquals("algal_2", outputter.bandName); assertEquals(3, outputter.pixels); assertEquals(0.8045, outputter.maximum, 1E-4); assertEquals(0.7216, outputter.minimum, 1E-4); assertEquals(0.7672, outputter.average, 1E-4); assertEquals(0.7758, outputter.median, 1E-4); assertEquals(0.0421, outputter.sigma, 1E-4); assertEquals(2, outputter.percentiles.length); assertEquals(0.8044, outputter.percentiles[0], 1E-4); assertEquals(0.8044, outputter.percentiles[1], 1E-4); }
@Test public void testStatisticsOp_WithDifferentPercentiles() throws Exception { final StatisticsOp statisticsOp = createStatisticsOp(); final BandConfiguration bandConfiguration = new BandConfiguration(); bandConfiguration.sourceBandName = "algal_2"; statisticsOp.bandConfigurations = new BandConfiguration[]{bandConfiguration}; statisticsOp.sourceProducts = new Product[]{TestUtil.getTestProduct()}; statisticsOp.shapefile = new File(getClass().getResource("4_pixels.shp").getFile()); statisticsOp.percentiles = new int[]{20, 51, 90}; final MyOutputter outputter = new MyOutputter(); statisticsOp.statisticsOutputters.add(outputter); statisticsOp.initialize(); assertEquals("4_pixels.1", outputter.region); assertEquals("algal_2", outputter.bandName); assertEquals(4, outputter.pixels); assertEquals(0.804474, outputter.maximum, 1E-3); assertEquals(0.695857, outputter.minimum, 1E-3); assertEquals(0.749427, outputter.average, 1E-3); assertEquals(0.721552, outputter.median, 1E-3); assertEquals(0.049577, outputter.sigma, 1E-3); assertEquals(3, outputter.percentiles.length); assertEquals(0.6958565, outputter.percentiles[0], 1E-3); assertEquals(0.775825, outputter.percentiles[1], 1E-3); assertEquals(0.804474, outputter.percentiles[2], 1E-3); }