@Test public void testRunInvalidQuery() throws Exception { commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool")); commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool")); String[] args = { "-i", getIndexPath1AsFile().toString(), "-o", seqFilesOutputPath.toString(), "-id", idField, "-f", StringUtils.join(fields, SequenceFilesFromLuceneStorageDriver.SEPARATOR_FIELDS), "-q", "invalid:query", "-xm", "sequential" }; driver.setConf(conf); driver.run(args); assertTrue(FileSystem.get(conf).exists(seqFilesOutputPath)); SequenceFileDirIterator<Writable, Writable> iter = new SequenceFileDirIterator<Writable, Writable>(seqFilesOutputPath, PathType.LIST, PathFilters.logsCRCFilter(), null, false, conf); assertFalse(Iterators.size(iter) > 0); }
@Override public int run(String[] args) throws Exception { addOutputOption(); addInputOption(); addOption(OPTION_ID_FIELD, "id", "The field in the index containing the id", true); addOption(OPTION_FIELD, "f", "The stored field(s) in the index containing text", true); addOption(OPTION_QUERY, "q", "(Optional) Lucene query. Defaults to " + DEFAULT_QUERY.getClass().getSimpleName()); addOption(OPTION_MAX_HITS, "n", "(Optional) Max hits. Defaults to " + DEFAULT_MAX_HITS); addOption(DefaultOptionCreator.methodOption().create()); if (parseArguments(args) == null) { return -1; } Configuration configuration = getConf(); String[] paths = getInputPath().toString().split(","); List<Path> indexPaths = Lists.newArrayList(); for (String path : paths) { indexPaths.add(new Path(path)); } Path sequenceFilesOutputPath = getOutputPath(); String idField = getOption(OPTION_ID_FIELD); String fields = getOption(OPTION_FIELD); LuceneStorageConfiguration lucene2SeqConf = newLucene2SeqConfiguration(configuration, indexPaths, sequenceFilesOutputPath, idField, asList(fields.split(SEPARATOR_FIELDS))); Query query = DEFAULT_QUERY; if (hasOption(OPTION_QUERY)) { try { String queryString = COMPILE.matcher(getOption(OPTION_QUERY)).replaceAll(""); QueryParser queryParser = new QueryParser(Version.LUCENE_46, queryString, new StandardAnalyzer(Version.LUCENE_46)); query = queryParser.parse(queryString); } catch (ParseException e) { throw new IllegalArgumentException(e.getMessage(), e); } } lucene2SeqConf.setQuery(query); int maxHits = DEFAULT_MAX_HITS; if (hasOption(OPTION_MAX_HITS)) { String maxHitsString = getOption(OPTION_MAX_HITS); maxHits = Integer.valueOf(maxHitsString); } lucene2SeqConf.setMaxHits(maxHits); if (hasOption(DefaultOptionCreator.METHOD_OPTION) && getOption(DefaultOptionCreator.METHOD_OPTION).equals("sequential")) { new SequenceFilesFromLuceneStorage().run(lucene2SeqConf); } else { new SequenceFilesFromLuceneStorageMRJob().run(lucene2SeqConf); } return 0; }
SequenceFilesFromLuceneStorageDriver extends AbstractJob { @Override public int run(String[] args) throws Exception { addOutputOption(); addInputOption(); addOption(OPTION_ID_FIELD, "id", "The field in the index containing the id", true); addOption(OPTION_FIELD, "f", "The stored field(s) in the index containing text", true); addOption(OPTION_QUERY, "q", "(Optional) Lucene query. Defaults to " + DEFAULT_QUERY.getClass().getSimpleName()); addOption(OPTION_MAX_HITS, "n", "(Optional) Max hits. Defaults to " + DEFAULT_MAX_HITS); addOption(DefaultOptionCreator.methodOption().create()); if (parseArguments(args) == null) { return -1; } Configuration configuration = getConf(); String[] paths = getInputPath().toString().split(","); List<Path> indexPaths = Lists.newArrayList(); for (String path : paths) { indexPaths.add(new Path(path)); } Path sequenceFilesOutputPath = getOutputPath(); String idField = getOption(OPTION_ID_FIELD); String fields = getOption(OPTION_FIELD); LuceneStorageConfiguration lucene2SeqConf = newLucene2SeqConfiguration(configuration, indexPaths, sequenceFilesOutputPath, idField, asList(fields.split(SEPARATOR_FIELDS))); Query query = DEFAULT_QUERY; if (hasOption(OPTION_QUERY)) { try { String queryString = COMPILE.matcher(getOption(OPTION_QUERY)).replaceAll(""); QueryParser queryParser = new QueryParser(Version.LUCENE_46, queryString, new StandardAnalyzer(Version.LUCENE_46)); query = queryParser.parse(queryString); } catch (ParseException e) { throw new IllegalArgumentException(e.getMessage(), e); } } lucene2SeqConf.setQuery(query); int maxHits = DEFAULT_MAX_HITS; if (hasOption(OPTION_MAX_HITS)) { String maxHitsString = getOption(OPTION_MAX_HITS); maxHits = Integer.valueOf(maxHitsString); } lucene2SeqConf.setMaxHits(maxHits); if (hasOption(DefaultOptionCreator.METHOD_OPTION) && getOption(DefaultOptionCreator.METHOD_OPTION).equals("sequential")) { new SequenceFilesFromLuceneStorage().run(lucene2SeqConf); } else { new SequenceFilesFromLuceneStorageMRJob().run(lucene2SeqConf); } return 0; } }
SequenceFilesFromLuceneStorageDriver extends AbstractJob { @Override public int run(String[] args) throws Exception { addOutputOption(); addInputOption(); addOption(OPTION_ID_FIELD, "id", "The field in the index containing the id", true); addOption(OPTION_FIELD, "f", "The stored field(s) in the index containing text", true); addOption(OPTION_QUERY, "q", "(Optional) Lucene query. Defaults to " + DEFAULT_QUERY.getClass().getSimpleName()); addOption(OPTION_MAX_HITS, "n", "(Optional) Max hits. Defaults to " + DEFAULT_MAX_HITS); addOption(DefaultOptionCreator.methodOption().create()); if (parseArguments(args) == null) { return -1; } Configuration configuration = getConf(); String[] paths = getInputPath().toString().split(","); List<Path> indexPaths = Lists.newArrayList(); for (String path : paths) { indexPaths.add(new Path(path)); } Path sequenceFilesOutputPath = getOutputPath(); String idField = getOption(OPTION_ID_FIELD); String fields = getOption(OPTION_FIELD); LuceneStorageConfiguration lucene2SeqConf = newLucene2SeqConfiguration(configuration, indexPaths, sequenceFilesOutputPath, idField, asList(fields.split(SEPARATOR_FIELDS))); Query query = DEFAULT_QUERY; if (hasOption(OPTION_QUERY)) { try { String queryString = COMPILE.matcher(getOption(OPTION_QUERY)).replaceAll(""); QueryParser queryParser = new QueryParser(Version.LUCENE_46, queryString, new StandardAnalyzer(Version.LUCENE_46)); query = queryParser.parse(queryString); } catch (ParseException e) { throw new IllegalArgumentException(e.getMessage(), e); } } lucene2SeqConf.setQuery(query); int maxHits = DEFAULT_MAX_HITS; if (hasOption(OPTION_MAX_HITS)) { String maxHitsString = getOption(OPTION_MAX_HITS); maxHits = Integer.valueOf(maxHitsString); } lucene2SeqConf.setMaxHits(maxHits); if (hasOption(DefaultOptionCreator.METHOD_OPTION) && getOption(DefaultOptionCreator.METHOD_OPTION).equals("sequential")) { new SequenceFilesFromLuceneStorage().run(lucene2SeqConf); } else { new SequenceFilesFromLuceneStorageMRJob().run(lucene2SeqConf); } return 0; }  }
SequenceFilesFromLuceneStorageDriver extends AbstractJob { @Override public int run(String[] args) throws Exception { addOutputOption(); addInputOption(); addOption(OPTION_ID_FIELD, "id", "The field in the index containing the id", true); addOption(OPTION_FIELD, "f", "The stored field(s) in the index containing text", true); addOption(OPTION_QUERY, "q", "(Optional) Lucene query. Defaults to " + DEFAULT_QUERY.getClass().getSimpleName()); addOption(OPTION_MAX_HITS, "n", "(Optional) Max hits. Defaults to " + DEFAULT_MAX_HITS); addOption(DefaultOptionCreator.methodOption().create()); if (parseArguments(args) == null) { return -1; } Configuration configuration = getConf(); String[] paths = getInputPath().toString().split(","); List<Path> indexPaths = Lists.newArrayList(); for (String path : paths) { indexPaths.add(new Path(path)); } Path sequenceFilesOutputPath = getOutputPath(); String idField = getOption(OPTION_ID_FIELD); String fields = getOption(OPTION_FIELD); LuceneStorageConfiguration lucene2SeqConf = newLucene2SeqConfiguration(configuration, indexPaths, sequenceFilesOutputPath, idField, asList(fields.split(SEPARATOR_FIELDS))); Query query = DEFAULT_QUERY; if (hasOption(OPTION_QUERY)) { try { String queryString = COMPILE.matcher(getOption(OPTION_QUERY)).replaceAll(""); QueryParser queryParser = new QueryParser(Version.LUCENE_46, queryString, new StandardAnalyzer(Version.LUCENE_46)); query = queryParser.parse(queryString); } catch (ParseException e) { throw new IllegalArgumentException(e.getMessage(), e); } } lucene2SeqConf.setQuery(query); int maxHits = DEFAULT_MAX_HITS; if (hasOption(OPTION_MAX_HITS)) { String maxHitsString = getOption(OPTION_MAX_HITS); maxHits = Integer.valueOf(maxHitsString); } lucene2SeqConf.setMaxHits(maxHits); if (hasOption(DefaultOptionCreator.METHOD_OPTION) && getOption(DefaultOptionCreator.METHOD_OPTION).equals("sequential")) { new SequenceFilesFromLuceneStorage().run(lucene2SeqConf); } else { new SequenceFilesFromLuceneStorageMRJob().run(lucene2SeqConf); } return 0; }  static void main(String[] args); @Override int run(String[] args); LuceneStorageConfiguration newLucene2SeqConfiguration(Configuration configuration,
                                                               List<Path> indexPaths,
                                                               Path sequenceFilesOutputPath,
                                                               String idField,
                                                               List<String> fields); }
SequenceFilesFromLuceneStorageDriver extends AbstractJob { @Override public int run(String[] args) throws Exception { addOutputOption(); addInputOption(); addOption(OPTION_ID_FIELD, "id", "The field in the index containing the id", true); addOption(OPTION_FIELD, "f", "The stored field(s) in the index containing text", true); addOption(OPTION_QUERY, "q", "(Optional) Lucene query. Defaults to " + DEFAULT_QUERY.getClass().getSimpleName()); addOption(OPTION_MAX_HITS, "n", "(Optional) Max hits. Defaults to " + DEFAULT_MAX_HITS); addOption(DefaultOptionCreator.methodOption().create()); if (parseArguments(args) == null) { return -1; } Configuration configuration = getConf(); String[] paths = getInputPath().toString().split(","); List<Path> indexPaths = Lists.newArrayList(); for (String path : paths) { indexPaths.add(new Path(path)); } Path sequenceFilesOutputPath = getOutputPath(); String idField = getOption(OPTION_ID_FIELD); String fields = getOption(OPTION_FIELD); LuceneStorageConfiguration lucene2SeqConf = newLucene2SeqConfiguration(configuration, indexPaths, sequenceFilesOutputPath, idField, asList(fields.split(SEPARATOR_FIELDS))); Query query = DEFAULT_QUERY; if (hasOption(OPTION_QUERY)) { try { String queryString = COMPILE.matcher(getOption(OPTION_QUERY)).replaceAll(""); QueryParser queryParser = new QueryParser(Version.LUCENE_46, queryString, new StandardAnalyzer(Version.LUCENE_46)); query = queryParser.parse(queryString); } catch (ParseException e) { throw new IllegalArgumentException(e.getMessage(), e); } } lucene2SeqConf.setQuery(query); int maxHits = DEFAULT_MAX_HITS; if (hasOption(OPTION_MAX_HITS)) { String maxHitsString = getOption(OPTION_MAX_HITS); maxHits = Integer.valueOf(maxHitsString); } lucene2SeqConf.setMaxHits(maxHits); if (hasOption(DefaultOptionCreator.METHOD_OPTION) && getOption(DefaultOptionCreator.METHOD_OPTION).equals("sequential")) { new SequenceFilesFromLuceneStorage().run(lucene2SeqConf); } else { new SequenceFilesFromLuceneStorageMRJob().run(lucene2SeqConf); } return 0; }  static void main(String[] args); @Override int run(String[] args); LuceneStorageConfiguration newLucene2SeqConfiguration(Configuration configuration,
                                                               List<Path> indexPaths,
                                                               Path sequenceFilesOutputPath,
                                                               String idField,
                                                               List<String> fields);  }