@Test public void testBad() throws Exception { Configuration conf = getDefaultRegexIngestMapperConfiguration(); conf.unset(RegexIngestMapper.REGEX); assertMapperConfigurationFailsWithMessage(conf, "com.lucidworks.hadoop.ingest.RegexIngestMapper.regex property must not be null or empty"); conf.set(RegexIngestMapper.REGEX, ""); assertMapperConfigurationFailsWithMessage(conf, "com.lucidworks.hadoop.ingest.RegexIngestMapper.regex property must not be null or empty"); conf.set(RegexIngestMapper.REGEX, "\\w+"); conf.set(RegexIngestMapper.GROUPS_TO_FIELDS, ""); assertMapperConfigurationFailsWithMessage(conf, "com.lucidworks.hadoop.ingest.RegexIngestMapper.groups_to_fields property must not be null or empty"); conf.set(RegexIngestMapper.REGEX, "\\w+"); conf.set(RegexIngestMapper.GROUPS_TO_FIELDS, "0"); assertMapperConfigurationFailsWithMessage(conf, "Malformed com.lucidworks.hadoop.ingest.RegexIngestMapper.groups_to_fields"); conf.set(RegexIngestMapper.GROUPS_TO_FIELDS, "0=foo,1"); assertMapperConfigurationFailsWithMessage(conf, "Malformed com.lucidworks.hadoop.ingest.RegexIngestMapper.groups_to_fields"); RegexIngestMapper mapper = new RegexIngestMapper(); LWDocument[] docs = mapper.toDocuments(null, null, null, null); Assert.assertNull(docs); }
@Override public LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter, Configuration conf) throws IOException { if (key != null && value != null) { LWDocument doc = createDocument(key.toString() + "-" + System.currentTimeMillis(), null); Matcher matcher = regex.matcher(value.toString()); if (matcher != null) { if (match) { if (matcher.matches()) { processMatch(doc, matcher); } } else { while (matcher.find()) { processMatch(doc, matcher); reporter.progress(); } } } FileSplit fileSplit = (FileSplit) reporter.getInputSplit(); String originalLogFilePath = fileSplit.getPath().toUri().getPath(); doc.addField(FIELD_PATH, originalLogFilePath); String docId = originalLogFilePath + "-" + doc.getId(); doc.setId(docId); return new LWDocument[] {doc}; } return null; }
RegexIngestMapper extends AbstractIngestMapper<Writable, Writable> { @Override public LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter, Configuration conf) throws IOException { if (key != null && value != null) { LWDocument doc = createDocument(key.toString() + "-" + System.currentTimeMillis(), null); Matcher matcher = regex.matcher(value.toString()); if (matcher != null) { if (match) { if (matcher.matches()) { processMatch(doc, matcher); } } else { while (matcher.find()) { processMatch(doc, matcher); reporter.progress(); } } } FileSplit fileSplit = (FileSplit) reporter.getInputSplit(); String originalLogFilePath = fileSplit.getPath().toUri().getPath(); doc.addField(FIELD_PATH, originalLogFilePath); String docId = originalLogFilePath + "-" + doc.getId(); doc.setId(docId); return new LWDocument[] {doc}; } return null; } }
RegexIngestMapper extends AbstractIngestMapper<Writable, Writable> { @Override public LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter, Configuration conf) throws IOException { if (key != null && value != null) { LWDocument doc = createDocument(key.toString() + "-" + System.currentTimeMillis(), null); Matcher matcher = regex.matcher(value.toString()); if (matcher != null) { if (match) { if (matcher.matches()) { processMatch(doc, matcher); } } else { while (matcher.find()) { processMatch(doc, matcher); reporter.progress(); } } } FileSplit fileSplit = (FileSplit) reporter.getInputSplit(); String originalLogFilePath = fileSplit.getPath().toUri().getPath(); doc.addField(FIELD_PATH, originalLogFilePath); String docId = originalLogFilePath + "-" + doc.getId(); doc.setId(docId); return new LWDocument[] {doc}; } return null; }  }
RegexIngestMapper extends AbstractIngestMapper<Writable, Writable> { @Override public LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter, Configuration conf) throws IOException { if (key != null && value != null) { LWDocument doc = createDocument(key.toString() + "-" + System.currentTimeMillis(), null); Matcher matcher = regex.matcher(value.toString()); if (matcher != null) { if (match) { if (matcher.matches()) { processMatch(doc, matcher); } } else { while (matcher.find()) { processMatch(doc, matcher); reporter.progress(); } } } FileSplit fileSplit = (FileSplit) reporter.getInputSplit(); String originalLogFilePath = fileSplit.getPath().toUri().getPath(); doc.addField(FIELD_PATH, originalLogFilePath); String docId = originalLogFilePath + "-" + doc.getId(); doc.setId(docId); return new LWDocument[] {doc}; } return null; }  @Override AbstractJobFixture getFixture(); @Override void configure(JobConf conf); @Override LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter,
      Configuration conf); }
RegexIngestMapper extends AbstractIngestMapper<Writable, Writable> { @Override public LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter, Configuration conf) throws IOException { if (key != null && value != null) { LWDocument doc = createDocument(key.toString() + "-" + System.currentTimeMillis(), null); Matcher matcher = regex.matcher(value.toString()); if (matcher != null) { if (match) { if (matcher.matches()) { processMatch(doc, matcher); } } else { while (matcher.find()) { processMatch(doc, matcher); reporter.progress(); } } } FileSplit fileSplit = (FileSplit) reporter.getInputSplit(); String originalLogFilePath = fileSplit.getPath().toUri().getPath(); doc.addField(FIELD_PATH, originalLogFilePath); String docId = originalLogFilePath + "-" + doc.getId(); doc.setId(docId); return new LWDocument[] {doc}; } return null; }  @Override AbstractJobFixture getFixture(); @Override void configure(JobConf conf); @Override LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter,
      Configuration conf); static final String REGEX; static final String GROUPS_TO_FIELDS; static final String REGEX_MATCH; static final String FIELD_PATH; }