@Test public void testRun2Directories() throws Exception { commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(0, 500)); commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(1000, 1500)); commitDocuments(getDirectory(getIndexPath2AsFile()), docs.subList(500, 1000)); commitDocuments(getDirectory(getIndexPath2AsFile()), docs.subList(1500, 2000)); commitDocuments(getDirectory(getIndexPath1AsFile()), misshapenDocs); lucene2Seq.run(lucene2SeqConf); Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator(); Map<String, Text> map = Maps.newHashMap(); while (iterator.hasNext()) { Pair<Text, Text> next = iterator.next(); map.put(next.getFirst().toString(), next.getSecond()); } assertEquals(docs.size() + misshapenDocs.size(), map.size()); for (TestDocument doc : docs) { Text value = map.get(doc.getId()); assertNotNull(value); assertEquals(value.toString(), doc.getField()); } for (TestDocument doc : misshapenDocs) { Text value = map.get(doc.getId()); assertNotNull(value); assertEquals(value.toString(), doc.getField()); } }
public void run(final LuceneStorageConfiguration lucene2seqConf) throws IOException { List<Path> indexPaths = lucene2seqConf.getIndexPaths(); int processedDocs = 0; for (Path indexPath : indexPaths) { Directory directory = FSDirectory.open(new File(indexPath.toUri().getPath())); IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); LuceneIndexHelper.fieldShouldExistInIndex(reader, lucene2seqConf.getIdField()); for (String field : lucene2seqConf.getFields()) { LuceneIndexHelper.fieldShouldExistInIndex(reader, field); } Configuration configuration = lucene2seqConf.getConfiguration(); FileSystem fileSystem = FileSystem.get(configuration); Path sequenceFilePath = new Path(lucene2seqConf.getSequenceFilesOutputPath(), indexPath.getName()); final SequenceFile.Writer sequenceFileWriter = new SequenceFile.Writer(fileSystem, configuration, sequenceFilePath, Text.class, Text.class); SeqFileWriterCollector writerCollector = new SeqFileWriterCollector(lucene2seqConf, sequenceFileWriter, processedDocs); searcher.search(lucene2seqConf.getQuery(), writerCollector); log.info("Wrote " + writerCollector.processedDocs + " documents in " + sequenceFilePath.toUri()); processedDocs = writerCollector.processedDocs; Closeables.close(sequenceFileWriter, false); directory.close(); reader.close(); } }
SequenceFilesFromLuceneStorage { public void run(final LuceneStorageConfiguration lucene2seqConf) throws IOException { List<Path> indexPaths = lucene2seqConf.getIndexPaths(); int processedDocs = 0; for (Path indexPath : indexPaths) { Directory directory = FSDirectory.open(new File(indexPath.toUri().getPath())); IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); LuceneIndexHelper.fieldShouldExistInIndex(reader, lucene2seqConf.getIdField()); for (String field : lucene2seqConf.getFields()) { LuceneIndexHelper.fieldShouldExistInIndex(reader, field); } Configuration configuration = lucene2seqConf.getConfiguration(); FileSystem fileSystem = FileSystem.get(configuration); Path sequenceFilePath = new Path(lucene2seqConf.getSequenceFilesOutputPath(), indexPath.getName()); final SequenceFile.Writer sequenceFileWriter = new SequenceFile.Writer(fileSystem, configuration, sequenceFilePath, Text.class, Text.class); SeqFileWriterCollector writerCollector = new SeqFileWriterCollector(lucene2seqConf, sequenceFileWriter, processedDocs); searcher.search(lucene2seqConf.getQuery(), writerCollector); log.info("Wrote " + writerCollector.processedDocs + " documents in " + sequenceFilePath.toUri()); processedDocs = writerCollector.processedDocs; Closeables.close(sequenceFileWriter, false); directory.close(); reader.close(); } } }
SequenceFilesFromLuceneStorage { public void run(final LuceneStorageConfiguration lucene2seqConf) throws IOException { List<Path> indexPaths = lucene2seqConf.getIndexPaths(); int processedDocs = 0; for (Path indexPath : indexPaths) { Directory directory = FSDirectory.open(new File(indexPath.toUri().getPath())); IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); LuceneIndexHelper.fieldShouldExistInIndex(reader, lucene2seqConf.getIdField()); for (String field : lucene2seqConf.getFields()) { LuceneIndexHelper.fieldShouldExistInIndex(reader, field); } Configuration configuration = lucene2seqConf.getConfiguration(); FileSystem fileSystem = FileSystem.get(configuration); Path sequenceFilePath = new Path(lucene2seqConf.getSequenceFilesOutputPath(), indexPath.getName()); final SequenceFile.Writer sequenceFileWriter = new SequenceFile.Writer(fileSystem, configuration, sequenceFilePath, Text.class, Text.class); SeqFileWriterCollector writerCollector = new SeqFileWriterCollector(lucene2seqConf, sequenceFileWriter, processedDocs); searcher.search(lucene2seqConf.getQuery(), writerCollector); log.info("Wrote " + writerCollector.processedDocs + " documents in " + sequenceFilePath.toUri()); processedDocs = writerCollector.processedDocs; Closeables.close(sequenceFileWriter, false); directory.close(); reader.close(); } }  }
SequenceFilesFromLuceneStorage { public void run(final LuceneStorageConfiguration lucene2seqConf) throws IOException { List<Path> indexPaths = lucene2seqConf.getIndexPaths(); int processedDocs = 0; for (Path indexPath : indexPaths) { Directory directory = FSDirectory.open(new File(indexPath.toUri().getPath())); IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); LuceneIndexHelper.fieldShouldExistInIndex(reader, lucene2seqConf.getIdField()); for (String field : lucene2seqConf.getFields()) { LuceneIndexHelper.fieldShouldExistInIndex(reader, field); } Configuration configuration = lucene2seqConf.getConfiguration(); FileSystem fileSystem = FileSystem.get(configuration); Path sequenceFilePath = new Path(lucene2seqConf.getSequenceFilesOutputPath(), indexPath.getName()); final SequenceFile.Writer sequenceFileWriter = new SequenceFile.Writer(fileSystem, configuration, sequenceFilePath, Text.class, Text.class); SeqFileWriterCollector writerCollector = new SeqFileWriterCollector(lucene2seqConf, sequenceFileWriter, processedDocs); searcher.search(lucene2seqConf.getQuery(), writerCollector); log.info("Wrote " + writerCollector.processedDocs + " documents in " + sequenceFilePath.toUri()); processedDocs = writerCollector.processedDocs; Closeables.close(sequenceFileWriter, false); directory.close(); reader.close(); } }  void run(final LuceneStorageConfiguration lucene2seqConf); }
SequenceFilesFromLuceneStorage { public void run(final LuceneStorageConfiguration lucene2seqConf) throws IOException { List<Path> indexPaths = lucene2seqConf.getIndexPaths(); int processedDocs = 0; for (Path indexPath : indexPaths) { Directory directory = FSDirectory.open(new File(indexPath.toUri().getPath())); IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); LuceneIndexHelper.fieldShouldExistInIndex(reader, lucene2seqConf.getIdField()); for (String field : lucene2seqConf.getFields()) { LuceneIndexHelper.fieldShouldExistInIndex(reader, field); } Configuration configuration = lucene2seqConf.getConfiguration(); FileSystem fileSystem = FileSystem.get(configuration); Path sequenceFilePath = new Path(lucene2seqConf.getSequenceFilesOutputPath(), indexPath.getName()); final SequenceFile.Writer sequenceFileWriter = new SequenceFile.Writer(fileSystem, configuration, sequenceFilePath, Text.class, Text.class); SeqFileWriterCollector writerCollector = new SeqFileWriterCollector(lucene2seqConf, sequenceFileWriter, processedDocs); searcher.search(lucene2seqConf.getQuery(), writerCollector); log.info("Wrote " + writerCollector.processedDocs + " documents in " + sequenceFilePath.toUri()); processedDocs = writerCollector.processedDocs; Closeables.close(sequenceFileWriter, false); directory.close(); reader.close(); } }  void run(final LuceneStorageConfiguration lucene2seqConf);  }