@Test @Category(UnitTest.class) public void getSplitsNoBounds() throws Exception { TaskAttemptContext mockContext = createDefaultTaskAttemptContext(); List<InputSplit> result = subject.getSplits(mockContext); Assert.assertEquals(1, result.size()); verifySplitsAreEqual(result); }
@Override public List<InputSplit> getSplits(JobContext context) throws IOException { long start = System.currentTimeMillis(); Configuration conf = context.getConfiguration(); ImageInputFormatContext ifContext = ImageInputFormatContext.load(conf); int zoom = ifContext.getZoomLevel(); int tilesize = ifContext.getTileSize(); HdfsMrsImageDataProvider dp = createHdfsMrsImageDataProvider(context.getConfiguration()); Path inputWithZoom = new Path(dp.getResourcePath(true), "" + zoom); MrsPyramidMetadataReader metadataReader = dp.getMetadataReader(); MrsPyramidMetadata metadata = metadataReader.read(); org.mrgeo.hdfs.tile.FileSplit fsplit = createFileSplit(); fsplit.readSplits(inputWithZoom); FileSplitInfo[] splits = (FileSplitInfo[]) fsplit.getSplits(); List<InputSplit> result = new ArrayList<>(splits.length); Bounds requestedBounds = ifContext.getBounds(); for (FileSplitInfo split : splits) { Path part = new Path(inputWithZoom, split.getName()); Path dataFile = new Path(part, MapFile.DATA_FILE_NAME); long endTileId = split.getEndId(); long startTileId = split.getStartId(); if (requestedBounds != null) { Tile startTile = TMSUtils.tileid(startTileId, zoom); Bounds startTileBounds = TMSUtils.tileBounds(startTile, zoom, tilesize); Tile endTile = TMSUtils.tileid(endTileId, zoom); Bounds endTileBounds = TMSUtils.tileBounds(endTile, zoom, tilesize); if (startTileBounds.s > requestedBounds.n || endTileBounds.n < requestedBounds.s) { } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } long end = System.currentTimeMillis(); log.info("Time to generate splits: " + (end - start) + " ms"); return result; }
HdfsMrsPyramidInputFormat extends InputFormat<TileIdWritable, RasterWritable> { @Override public List<InputSplit> getSplits(JobContext context) throws IOException { long start = System.currentTimeMillis(); Configuration conf = context.getConfiguration(); ImageInputFormatContext ifContext = ImageInputFormatContext.load(conf); int zoom = ifContext.getZoomLevel(); int tilesize = ifContext.getTileSize(); HdfsMrsImageDataProvider dp = createHdfsMrsImageDataProvider(context.getConfiguration()); Path inputWithZoom = new Path(dp.getResourcePath(true), "" + zoom); MrsPyramidMetadataReader metadataReader = dp.getMetadataReader(); MrsPyramidMetadata metadata = metadataReader.read(); org.mrgeo.hdfs.tile.FileSplit fsplit = createFileSplit(); fsplit.readSplits(inputWithZoom); FileSplitInfo[] splits = (FileSplitInfo[]) fsplit.getSplits(); List<InputSplit> result = new ArrayList<>(splits.length); Bounds requestedBounds = ifContext.getBounds(); for (FileSplitInfo split : splits) { Path part = new Path(inputWithZoom, split.getName()); Path dataFile = new Path(part, MapFile.DATA_FILE_NAME); long endTileId = split.getEndId(); long startTileId = split.getStartId(); if (requestedBounds != null) { Tile startTile = TMSUtils.tileid(startTileId, zoom); Bounds startTileBounds = TMSUtils.tileBounds(startTile, zoom, tilesize); Tile endTile = TMSUtils.tileid(endTileId, zoom); Bounds endTileBounds = TMSUtils.tileBounds(endTile, zoom, tilesize); if (startTileBounds.s > requestedBounds.n || endTileBounds.n < requestedBounds.s) { } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } long end = System.currentTimeMillis(); log.info("Time to generate splits: " + (end - start) + " ms"); return result; } }
HdfsMrsPyramidInputFormat extends InputFormat<TileIdWritable, RasterWritable> { @Override public List<InputSplit> getSplits(JobContext context) throws IOException { long start = System.currentTimeMillis(); Configuration conf = context.getConfiguration(); ImageInputFormatContext ifContext = ImageInputFormatContext.load(conf); int zoom = ifContext.getZoomLevel(); int tilesize = ifContext.getTileSize(); HdfsMrsImageDataProvider dp = createHdfsMrsImageDataProvider(context.getConfiguration()); Path inputWithZoom = new Path(dp.getResourcePath(true), "" + zoom); MrsPyramidMetadataReader metadataReader = dp.getMetadataReader(); MrsPyramidMetadata metadata = metadataReader.read(); org.mrgeo.hdfs.tile.FileSplit fsplit = createFileSplit(); fsplit.readSplits(inputWithZoom); FileSplitInfo[] splits = (FileSplitInfo[]) fsplit.getSplits(); List<InputSplit> result = new ArrayList<>(splits.length); Bounds requestedBounds = ifContext.getBounds(); for (FileSplitInfo split : splits) { Path part = new Path(inputWithZoom, split.getName()); Path dataFile = new Path(part, MapFile.DATA_FILE_NAME); long endTileId = split.getEndId(); long startTileId = split.getStartId(); if (requestedBounds != null) { Tile startTile = TMSUtils.tileid(startTileId, zoom); Bounds startTileBounds = TMSUtils.tileBounds(startTile, zoom, tilesize); Tile endTile = TMSUtils.tileid(endTileId, zoom); Bounds endTileBounds = TMSUtils.tileBounds(endTile, zoom, tilesize); if (startTileBounds.s > requestedBounds.n || endTileBounds.n < requestedBounds.s) { } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } long end = System.currentTimeMillis(); log.info("Time to generate splits: " + (end - start) + " ms"); return result; } HdfsMrsPyramidInputFormat();  HdfsMrsPyramidInputFormat(String input); }
HdfsMrsPyramidInputFormat extends InputFormat<TileIdWritable, RasterWritable> { @Override public List<InputSplit> getSplits(JobContext context) throws IOException { long start = System.currentTimeMillis(); Configuration conf = context.getConfiguration(); ImageInputFormatContext ifContext = ImageInputFormatContext.load(conf); int zoom = ifContext.getZoomLevel(); int tilesize = ifContext.getTileSize(); HdfsMrsImageDataProvider dp = createHdfsMrsImageDataProvider(context.getConfiguration()); Path inputWithZoom = new Path(dp.getResourcePath(true), "" + zoom); MrsPyramidMetadataReader metadataReader = dp.getMetadataReader(); MrsPyramidMetadata metadata = metadataReader.read(); org.mrgeo.hdfs.tile.FileSplit fsplit = createFileSplit(); fsplit.readSplits(inputWithZoom); FileSplitInfo[] splits = (FileSplitInfo[]) fsplit.getSplits(); List<InputSplit> result = new ArrayList<>(splits.length); Bounds requestedBounds = ifContext.getBounds(); for (FileSplitInfo split : splits) { Path part = new Path(inputWithZoom, split.getName()); Path dataFile = new Path(part, MapFile.DATA_FILE_NAME); long endTileId = split.getEndId(); long startTileId = split.getStartId(); if (requestedBounds != null) { Tile startTile = TMSUtils.tileid(startTileId, zoom); Bounds startTileBounds = TMSUtils.tileBounds(startTile, zoom, tilesize); Tile endTile = TMSUtils.tileid(endTileId, zoom); Bounds endTileBounds = TMSUtils.tileBounds(endTile, zoom, tilesize); if (startTileBounds.s > requestedBounds.n || endTileBounds.n < requestedBounds.s) { } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } long end = System.currentTimeMillis(); log.info("Time to generate splits: " + (end - start) + " ms"); return result; } HdfsMrsPyramidInputFormat();  HdfsMrsPyramidInputFormat(String input); static String getZoomName(HdfsMrsImageDataProvider dp,
    int zoomLevel); static void setInputInfo(Job job, String inputWithZoom); @Override RecordReader<TileIdWritable, RasterWritable> createRecordReader(InputSplit split,
    TaskAttemptContext context); @Override List<InputSplit> getSplits(JobContext context); }
HdfsMrsPyramidInputFormat extends InputFormat<TileIdWritable, RasterWritable> { @Override public List<InputSplit> getSplits(JobContext context) throws IOException { long start = System.currentTimeMillis(); Configuration conf = context.getConfiguration(); ImageInputFormatContext ifContext = ImageInputFormatContext.load(conf); int zoom = ifContext.getZoomLevel(); int tilesize = ifContext.getTileSize(); HdfsMrsImageDataProvider dp = createHdfsMrsImageDataProvider(context.getConfiguration()); Path inputWithZoom = new Path(dp.getResourcePath(true), "" + zoom); MrsPyramidMetadataReader metadataReader = dp.getMetadataReader(); MrsPyramidMetadata metadata = metadataReader.read(); org.mrgeo.hdfs.tile.FileSplit fsplit = createFileSplit(); fsplit.readSplits(inputWithZoom); FileSplitInfo[] splits = (FileSplitInfo[]) fsplit.getSplits(); List<InputSplit> result = new ArrayList<>(splits.length); Bounds requestedBounds = ifContext.getBounds(); for (FileSplitInfo split : splits) { Path part = new Path(inputWithZoom, split.getName()); Path dataFile = new Path(part, MapFile.DATA_FILE_NAME); long endTileId = split.getEndId(); long startTileId = split.getStartId(); if (requestedBounds != null) { Tile startTile = TMSUtils.tileid(startTileId, zoom); Bounds startTileBounds = TMSUtils.tileBounds(startTile, zoom, tilesize); Tile endTile = TMSUtils.tileid(endTileId, zoom); Bounds endTileBounds = TMSUtils.tileBounds(endTile, zoom, tilesize); if (startTileBounds.s > requestedBounds.n || endTileBounds.n < requestedBounds.s) { } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } else { result.add(new TiledInputSplit(new FileSplit(dataFile, 0, 0, null), startTileId, endTileId, zoom, metadata.getTilesize())); } } long end = System.currentTimeMillis(); log.info("Time to generate splits: " + (end - start) + " ms"); return result; } HdfsMrsPyramidInputFormat();  HdfsMrsPyramidInputFormat(String input); static String getZoomName(HdfsMrsImageDataProvider dp,
    int zoomLevel); static void setInputInfo(Job job, String inputWithZoom); @Override RecordReader<TileIdWritable, RasterWritable> createRecordReader(InputSplit split,
    TaskAttemptContext context); @Override List<InputSplit> getSplits(JobContext context);  }