@Test public void testRun() throws IOException { commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(0, 500)); commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(1000, 1500)); commitDocuments(getDirectory(getIndexPath2AsFile()), docs.subList(500, 1000)); commitDocuments(getDirectory(getIndexPath2AsFile()), docs.subList(1500, 2000)); commitDocuments(getDirectory(getIndexPath1AsFile()), misshapenDocs); lucene2seq.run(lucene2SeqConf); Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator(); Map<String, Text> map = Maps.newHashMap(); while (iterator.hasNext()) { Pair<Text, Text> next = iterator.next(); map.put(next.getFirst().toString(), next.getSecond()); } assertEquals(docs.size() + misshapenDocs.size(), map.size()); for (TestDocument doc : docs) { Text value = map.get(doc.getId()); assertNotNull(value); assertEquals(value.toString(), doc.getField()); } for (TestDocument doc : misshapenDocs) { Text value = map.get(doc.getId()); assertNotNull(value); assertEquals(value.toString(), doc.getField()); } }
public void run(LuceneStorageConfiguration lucene2seqConf) { try { Configuration configuration = lucene2seqConf.serialize(); Job job = new Job(configuration, "LuceneIndexToSequenceFiles: " + lucene2seqConf.getIndexPaths() + " -> M/R -> " + lucene2seqConf.getSequenceFilesOutputPath()); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapperClass(SequenceFilesFromLuceneStorageMapper.class); job.setInputFormatClass(LuceneSegmentInputFormat.class); FileInputFormat.setInputPaths(job, Joiner.on(',').skipNulls().join(lucene2seqConf.getIndexPaths().iterator())); FileOutputFormat.setOutputPath(job, lucene2seqConf.getSequenceFilesOutputPath()); job.setJarByClass(SequenceFilesFromLuceneStorageMRJob.class); job.setNumReduceTasks(0); job.waitForCompletion(true); } catch (IOException e) { throw new RuntimeException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } }
SequenceFilesFromLuceneStorageMRJob { public void run(LuceneStorageConfiguration lucene2seqConf) { try { Configuration configuration = lucene2seqConf.serialize(); Job job = new Job(configuration, "LuceneIndexToSequenceFiles: " + lucene2seqConf.getIndexPaths() + " -> M/R -> " + lucene2seqConf.getSequenceFilesOutputPath()); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapperClass(SequenceFilesFromLuceneStorageMapper.class); job.setInputFormatClass(LuceneSegmentInputFormat.class); FileInputFormat.setInputPaths(job, Joiner.on(',').skipNulls().join(lucene2seqConf.getIndexPaths().iterator())); FileOutputFormat.setOutputPath(job, lucene2seqConf.getSequenceFilesOutputPath()); job.setJarByClass(SequenceFilesFromLuceneStorageMRJob.class); job.setNumReduceTasks(0); job.waitForCompletion(true); } catch (IOException e) { throw new RuntimeException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } }
SequenceFilesFromLuceneStorageMRJob { public void run(LuceneStorageConfiguration lucene2seqConf) { try { Configuration configuration = lucene2seqConf.serialize(); Job job = new Job(configuration, "LuceneIndexToSequenceFiles: " + lucene2seqConf.getIndexPaths() + " -> M/R -> " + lucene2seqConf.getSequenceFilesOutputPath()); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapperClass(SequenceFilesFromLuceneStorageMapper.class); job.setInputFormatClass(LuceneSegmentInputFormat.class); FileInputFormat.setInputPaths(job, Joiner.on(',').skipNulls().join(lucene2seqConf.getIndexPaths().iterator())); FileOutputFormat.setOutputPath(job, lucene2seqConf.getSequenceFilesOutputPath()); job.setJarByClass(SequenceFilesFromLuceneStorageMRJob.class); job.setNumReduceTasks(0); job.waitForCompletion(true); } catch (IOException e) { throw new RuntimeException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } }  }
SequenceFilesFromLuceneStorageMRJob { public void run(LuceneStorageConfiguration lucene2seqConf) { try { Configuration configuration = lucene2seqConf.serialize(); Job job = new Job(configuration, "LuceneIndexToSequenceFiles: " + lucene2seqConf.getIndexPaths() + " -> M/R -> " + lucene2seqConf.getSequenceFilesOutputPath()); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapperClass(SequenceFilesFromLuceneStorageMapper.class); job.setInputFormatClass(LuceneSegmentInputFormat.class); FileInputFormat.setInputPaths(job, Joiner.on(',').skipNulls().join(lucene2seqConf.getIndexPaths().iterator())); FileOutputFormat.setOutputPath(job, lucene2seqConf.getSequenceFilesOutputPath()); job.setJarByClass(SequenceFilesFromLuceneStorageMRJob.class); job.setNumReduceTasks(0); job.waitForCompletion(true); } catch (IOException e) { throw new RuntimeException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } }  void run(LuceneStorageConfiguration lucene2seqConf); }
SequenceFilesFromLuceneStorageMRJob { public void run(LuceneStorageConfiguration lucene2seqConf) { try { Configuration configuration = lucene2seqConf.serialize(); Job job = new Job(configuration, "LuceneIndexToSequenceFiles: " + lucene2seqConf.getIndexPaths() + " -> M/R -> " + lucene2seqConf.getSequenceFilesOutputPath()); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapperClass(SequenceFilesFromLuceneStorageMapper.class); job.setInputFormatClass(LuceneSegmentInputFormat.class); FileInputFormat.setInputPaths(job, Joiner.on(',').skipNulls().join(lucene2seqConf.getIndexPaths().iterator())); FileOutputFormat.setOutputPath(job, lucene2seqConf.getSequenceFilesOutputPath()); job.setJarByClass(SequenceFilesFromLuceneStorageMRJob.class); job.setNumReduceTasks(0); job.waitForCompletion(true); } catch (IOException e) { throw new RuntimeException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } }  void run(LuceneStorageConfiguration lucene2seqConf);  }