LuceneIndexInputFormat extends InputFormat<IntWritable, T> { @Override public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException { loadConfig(HadoopCompat.getConfiguration(job)); PriorityQueue<LuceneIndexInputSplit> splits = findSplits(HadoopCompat.getConfiguration(job)); List<InputSplit> combinedSplits = combineSplits(splits, maxCombinedIndexSizePerSplit, maxNumIndexesPerSplit); return combinedSplits; }  PathFilter getIndexDirPathFilter(Configuration conf); @Override List<InputSplit> getSplits(JobContext job); static void setQueries(List<String> queries, Configuration conf); static List<String> getQueries(Configuration conf); static boolean queriesSet(Configuration conf); static void setInputPaths(List<Path> paths, Configuration conf); static Path[] getInputPaths(Configuration conf); static void setMaxCombinedIndexSizePerSplitBytes(long size, Configuration conf); static long getMaxCombinedIndexSizePerSplit(Configuration conf); static void setMaxNumIndexesPerSplit(long num, Configuration conf); static long getMaxNumIndexesPerSplit(Configuration conf); static final String QUERIES_KEY; static final String INPUT_PATHS_KEY; static final String MAX_NUM_INDEXES_PER_SPLIT_KEY; static final String MAX_COMBINED_INDEX_SIZE_PER_SPLIT_KEY; }
@Test public void testGetSplits() throws Exception { DummyLuceneInputFormat lif = new DummyLuceneInputFormat(); Configuration conf = new Configuration(); LuceneIndexInputFormat.setInputPaths( ImmutableList.of(new Path("src/test/resources/com/twitter/elephantbird" + "/mapreduce/input/sample_indexes/")), conf); LuceneIndexInputFormat.setMaxCombinedIndexSizePerSplitBytes(15L, conf); JobContext jobContext = createStrictMock(JobContext.class); expect(HadoopCompat.getConfiguration(jobContext)).andStubReturn(conf); replay(jobContext); List<InputSplit> splits = lif.getSplits(jobContext); LuceneIndexInputSplit split = (LuceneIndexInputSplit) splits.get(0); assertEquals(2, split.getIndexDirs().size()); assertTrue(split.getIndexDirs().get(0).toString().endsWith("sample_indexes/index-1")); assertTrue(split.getIndexDirs().get(1).toString() .endsWith("sample_indexes/more-indexes/index-3")); split = (LuceneIndexInputSplit) splits.get(1); assertEquals(1, split.getIndexDirs().size()); assertTrue(split.getIndexDirs().get(0).toString().endsWith("sample_indexes/index-2")); }