SequenceFilesFromLuceneStorage { public void run(final LuceneStorageConfiguration lucene2seqConf) throws IOException { List<Path> indexPaths = lucene2seqConf.getIndexPaths(); int processedDocs = 0; for (Path indexPath : indexPaths) { Directory directory = FSDirectory.open(new File(indexPath.toUri().getPath())); IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); LuceneIndexHelper.fieldShouldExistInIndex(reader, lucene2seqConf.getIdField()); for (String field : lucene2seqConf.getFields()) { LuceneIndexHelper.fieldShouldExistInIndex(reader, field); } Configuration configuration = lucene2seqConf.getConfiguration(); FileSystem fileSystem = FileSystem.get(configuration); Path sequenceFilePath = new Path(lucene2seqConf.getSequenceFilesOutputPath(), indexPath.getName()); final SequenceFile.Writer sequenceFileWriter = new SequenceFile.Writer(fileSystem, configuration, sequenceFilePath, Text.class, Text.class); SeqFileWriterCollector writerCollector = new SeqFileWriterCollector(lucene2seqConf, sequenceFileWriter, processedDocs); searcher.search(lucene2seqConf.getQuery(), writerCollector); log.info("Wrote " + writerCollector.processedDocs + " documents in " + sequenceFilePath.toUri()); processedDocs = writerCollector.processedDocs; Closeables.close(sequenceFileWriter, false); directory.close(); reader.close(); } }  void run(final LuceneStorageConfiguration lucene2seqConf);  }
@Test public void testRun2Directories() throws Exception { commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(0, 500)); commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(1000, 1500)); commitDocuments(getDirectory(getIndexPath2AsFile()), docs.subList(500, 1000)); commitDocuments(getDirectory(getIndexPath2AsFile()), docs.subList(1500, 2000)); commitDocuments(getDirectory(getIndexPath1AsFile()), misshapenDocs); lucene2Seq.run(lucene2SeqConf); Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator(); Map<String, Text> map = Maps.newHashMap(); while (iterator.hasNext()) { Pair<Text, Text> next = iterator.next(); map.put(next.getFirst().toString(), next.getSecond()); } assertEquals(docs.size() + misshapenDocs.size(), map.size()); for (TestDocument doc : docs) { Text value = map.get(doc.getId()); assertNotNull(value); assertEquals(value.toString(), doc.getField()); } for (TestDocument doc : misshapenDocs) { Text value = map.get(doc.getId()); assertNotNull(value); assertEquals(value.toString(), doc.getField()); } }
@Test(expected = IllegalArgumentException.class) public void testRun_UnstoredFields() throws IOException { commitDocuments(getDirectory(getIndexPath1AsFile()), new UnstoredFieldsDocument("5", "This is test document 5")); LuceneStorageConfiguration lucene2SeqConf = new LuceneStorageConfiguration(configuration, asList(getIndexPath1()), seqFilesOutputPath, SingleFieldDocument.ID_FIELD, asList(UnstoredFieldsDocument.FIELD, UnstoredFieldsDocument.UNSTORED_FIELD)); lucene2Seq.run(lucene2SeqConf); }
@Test public void testRunMaxHits() throws IOException { commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(0, 500)); commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(1000, 1500)); commitDocuments(getDirectory(getIndexPath2AsFile()), docs.subList(500, 1000)); commitDocuments(getDirectory(getIndexPath2AsFile()), docs.subList(1500, 2000)); lucene2SeqConf.setMaxHits(3); lucene2Seq.run(lucene2SeqConf); Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator(); assertTrue(iterator.hasNext()); iterator.next(); assertTrue(iterator.hasNext()); iterator.next(); assertTrue(iterator.hasNext()); iterator.next(); assertFalse(iterator.hasNext()); }
@Test public void testRunQuery() throws IOException { commitDocuments(getDirectory(getIndexPath1AsFile()), docs); LuceneStorageConfiguration lucene2SeqConf = new LuceneStorageConfiguration(configuration, asList(getIndexPath1()), seqFilesOutputPath, SingleFieldDocument.ID_FIELD, asList(SingleFieldDocument.FIELD)); Query query = new TermQuery(new Term(lucene2SeqConf.getFields().get(0), "599")); lucene2SeqConf.setQuery(query); lucene2Seq.run(lucene2SeqConf); Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator(); assertTrue(iterator.hasNext()); Pair<Text, Text> next = iterator.next(); assertTrue(next.getSecond().toString().contains("599")); assertFalse(iterator.hasNext()); }
@Test public void testRunMultipleFields() throws IOException { LuceneStorageConfiguration lucene2SeqConf = new LuceneStorageConfiguration(configuration, asList(getIndexPath1()), seqFilesOutputPath, SingleFieldDocument.ID_FIELD, asList(MultipleFieldsDocument.FIELD, MultipleFieldsDocument.FIELD1, MultipleFieldsDocument.FIELD2)); MultipleFieldsDocument multipleFieldsDocument1 = new MultipleFieldsDocument("1", "This is field 1-1", "This is field 1-2", "This is field 1-3"); MultipleFieldsDocument multipleFieldsDocument2 = new MultipleFieldsDocument("2", "This is field 2-1", "This is field 2-2", "This is field 2-3"); MultipleFieldsDocument multipleFieldsDocument3 = new MultipleFieldsDocument("3", "This is field 3-1", "This is field 3-2", "This is field 3-3"); commitDocuments(getDirectory(getIndexPath1AsFile()), multipleFieldsDocument1, multipleFieldsDocument2, multipleFieldsDocument3); lucene2Seq.run(lucene2SeqConf); Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator(); assertMultipleFieldsDocumentEquals(multipleFieldsDocument1, iterator.next()); assertMultipleFieldsDocumentEquals(multipleFieldsDocument2, iterator.next()); assertMultipleFieldsDocumentEquals(multipleFieldsDocument3, iterator.next()); }
@Test public void testRunNumericField() throws IOException { LuceneStorageConfiguration lucene2SeqConf = new LuceneStorageConfiguration(configuration, asList(getIndexPath1()), seqFilesOutputPath, SingleFieldDocument.ID_FIELD, asList(NumericFieldDocument.FIELD, NumericFieldDocument.NUMERIC_FIELD)); NumericFieldDocument doc1 = new NumericFieldDocument("1", "This is field 1", 100); NumericFieldDocument doc2 = new NumericFieldDocument("2", "This is field 2", 200); NumericFieldDocument doc3 = new NumericFieldDocument("3", "This is field 3", 300); commitDocuments(getDirectory(getIndexPath1AsFile()), doc1, doc2, doc3); lucene2Seq.run(lucene2SeqConf); Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator(); assertNumericFieldEquals(doc1, iterator.next()); assertNumericFieldEquals(doc2, iterator.next()); assertNumericFieldEquals(doc3, iterator.next()); }
@Test(expected = IllegalArgumentException.class) public void testNonExistingIdField() throws IOException { commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(0, 500)); lucene2SeqConf = new LuceneStorageConfiguration(configuration, asList(getIndexPath1()), seqFilesOutputPath, "nonExistingField", asList(SingleFieldDocument.FIELD)); lucene2Seq.run(lucene2SeqConf); }
@Test(expected = IllegalArgumentException.class) public void testNonExistingField() throws IOException { commitDocuments(getDirectory(getIndexPath1AsFile()), docs.subList(0, 500)); lucene2SeqConf = new LuceneStorageConfiguration(configuration, asList(getIndexPath1()), seqFilesOutputPath, SingleFieldDocument.ID_FIELD, asList(SingleFieldDocument.FIELD, "nonExistingField")); lucene2Seq.run(lucene2SeqConf); }
@Test(expected = IllegalArgumentException.class) public void testIndexedButNotStoredField() throws IOException { SingleFieldDocument document = new SingleFieldDocument("id", "field") { @Override public Document asLuceneDocument() { Document document = super.asLuceneDocument(); document.add(new TextField("indexed", "This text is indexed", Field.Store.NO)); return document; } }; commitDocuments(getDirectory(getIndexPath1AsFile()), document); lucene2SeqConf = new LuceneStorageConfiguration(configuration, asList(getIndexPath1()), seqFilesOutputPath, SingleFieldDocument.ID_FIELD, asList(SingleFieldDocument.FIELD, "indexed")); lucene2Seq.run(lucene2SeqConf); }