SequenceFilesFromLuceneStorageDriver extends AbstractJob { @Override public int run(String[] args) throws Exception { addOutputOption(); addInputOption(); addOption(OPTION_ID_FIELD, "id", "The field in the index containing the id", true); addOption(OPTION_FIELD, "f", "The stored field(s) in the index containing text", true); addOption(OPTION_QUERY, "q", "(Optional) Lucene query. Defaults to " + DEFAULT_QUERY.getClass().getSimpleName()); addOption(OPTION_MAX_HITS, "n", "(Optional) Max hits. Defaults to " + DEFAULT_MAX_HITS); addOption(DefaultOptionCreator.methodOption().create()); if (parseArguments(args) == null) { return -1; } Configuration configuration = getConf(); String[] paths = getInputPath().toString().split(","); List<Path> indexPaths = Lists.newArrayList(); for (String path : paths) { indexPaths.add(new Path(path)); } Path sequenceFilesOutputPath = getOutputPath(); String idField = getOption(OPTION_ID_FIELD); String fields = getOption(OPTION_FIELD); LuceneStorageConfiguration lucene2SeqConf = newLucene2SeqConfiguration(configuration, indexPaths, sequenceFilesOutputPath, idField, asList(fields.split(SEPARATOR_FIELDS))); Query query = DEFAULT_QUERY; if (hasOption(OPTION_QUERY)) { try { String queryString = COMPILE.matcher(getOption(OPTION_QUERY)).replaceAll(""); QueryParser queryParser = new QueryParser(Version.LUCENE_46, queryString, new StandardAnalyzer(Version.LUCENE_46)); query = queryParser.parse(queryString); } catch (ParseException e) { throw new IllegalArgumentException(e.getMessage(), e); } } lucene2SeqConf.setQuery(query); int maxHits = DEFAULT_MAX_HITS; if (hasOption(OPTION_MAX_HITS)) { String maxHitsString = getOption(OPTION_MAX_HITS); maxHits = Integer.valueOf(maxHitsString); } lucene2SeqConf.setMaxHits(maxHits); if (hasOption(DefaultOptionCreator.METHOD_OPTION) && getOption(DefaultOptionCreator.METHOD_OPTION).equals("sequential")) { new SequenceFilesFromLuceneStorage().run(lucene2SeqConf); } else { new SequenceFilesFromLuceneStorageMRJob().run(lucene2SeqConf); } return 0; }  static void main(String[] args); @Override int run(String[] args); LuceneStorageConfiguration newLucene2SeqConfiguration(Configuration configuration,
                                                               List<Path> indexPaths,
                                                               Path sequenceFilesOutputPath,
                                                               String idField,
                                                               List<String> fields);  }
@Test public void testRun() throws Exception { List<MultipleFieldsDocument> docs = asList(new MultipleFieldsDocument("123", "test 1", "test 2", "test 3")); commitDocuments(getDirectory(getIndexPath1AsFile()), docs.get(0)); String queryField = "queryfield"; String queryTerm = "queryterm"; String maxHits = "500"; String field1 = "field1"; String field2 = "field2"; String[] args = { "-i", getIndexPath1AsFile().toString(), "-o", seqFilesOutputPath.toString(), "-id", idField, "-f", field1 + "," + field2, "-q", queryField + ":" + queryTerm, "-n", maxHits, "-xm", "sequential" }; driver.setConf(conf); driver.run(args); assertEquals(1, lucene2SeqConf.getIndexPaths().size()); assertEquals(getIndexPath1().toUri().getPath(), lucene2SeqConf.getIndexPaths().get(0).toUri().getPath()); assertEquals(seqFilesOutputPath, lucene2SeqConf.getSequenceFilesOutputPath()); assertEquals(idField, lucene2SeqConf.getIdField()); assertEquals(asList(field1, field2), lucene2SeqConf.getFields()); assertTrue(lucene2SeqConf.getQuery() instanceof TermQuery); assertEquals(queryField, ((TermQuery) lucene2SeqConf.getQuery()).getTerm().field()); assertEquals(queryTerm, ((TermQuery) lucene2SeqConf.getQuery()).getTerm().text()); assertEquals(new Integer(maxHits), (Integer) lucene2SeqConf.getMaxHits()); }
@Test public void testRunOptionalArguments() throws Exception { commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool")); commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool")); String[] args = { "-i", getIndexPath1AsFile().toString(), "-o", seqFilesOutputPath.toString(), "-id", idField, "-f", StringUtils.join(fields, SequenceFilesFromLuceneStorageDriver.SEPARATOR_FIELDS) }; driver.setConf(conf); driver.run(args); assertEquals(1, lucene2SeqConf.getIndexPaths().size()); assertEquals(getIndexPath1().toUri().getPath(), lucene2SeqConf.getIndexPaths().get(0).toUri().getPath()); assertEquals(seqFilesOutputPath, lucene2SeqConf.getSequenceFilesOutputPath()); assertEquals(idField, lucene2SeqConf.getIdField()); assertEquals(fields, lucene2SeqConf.getFields()); assertEquals(conf, lucene2SeqConf.getConfiguration()); assertEquals(SequenceFilesFromLuceneStorageDriver.DEFAULT_QUERY, lucene2SeqConf.getQuery()); assertEquals(SequenceFilesFromLuceneStorageDriver.DEFAULT_MAX_HITS, lucene2SeqConf.getMaxHits()); }
@Test public void testRunInvalidQuery() throws Exception { commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool")); commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool")); String[] args = { "-i", getIndexPath1AsFile().toString(), "-o", seqFilesOutputPath.toString(), "-id", idField, "-f", StringUtils.join(fields, SequenceFilesFromLuceneStorageDriver.SEPARATOR_FIELDS), "-q", "invalid:query", "-xm", "sequential" }; driver.setConf(conf); driver.run(args); assertTrue(FileSystem.get(conf).exists(seqFilesOutputPath)); SequenceFileDirIterator<Writable, Writable> iter = new SequenceFileDirIterator<Writable, Writable>(seqFilesOutputPath, PathType.LIST, PathFilters.logsCRCFilter(), null, false, conf); assertFalse(Iterators.size(iter) > 0); }
@Test public void testHelp() throws Exception { driver = new SequenceFilesFromLuceneStorageDriver(); driver.run(new String[]{"--help"}); }