OmitIndexingFilter implements IndexingFilter { @Override public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { String realUrl = new String(url.getBytes()).substring(0, url.getLength()); Metadata metadata = parse.getData().getParseMeta(); if (omitIndexingFilterConfiguration.getFilteringType() == FilteringType.WHITELIST) { for (OmitIndexingFilterConfigurationEntry omitIndexingFilterConfigurationEntry : omitIndexingFilterConfiguration.getOmitIndexingFilterConfigurationEntryList()) { switch (omitIndexingFilterConfigurationEntry.getTarget()) { case URL: Pattern pattern = Pattern.compile(omitIndexingFilterConfigurationEntry.getRegex()); if (pattern.matcher(realUrl).matches()) { return doc; } break; case META_FIELD_PRESENT: if(metadata.get(FilterUtils.getNullSafe(omitIndexingFilterConfigurationEntry.getName(), "")) != null) { return doc; } break; } } return null; } else if (omitIndexingFilterConfiguration.getFilteringType() == FilteringType.BLACKLIST) { for (OmitIndexingFilterConfigurationEntry omitIndexingFilterConfigurationEntry : omitIndexingFilterConfiguration.getOmitIndexingFilterConfigurationEntryList()) { switch (omitIndexingFilterConfigurationEntry.getTarget()) { case URL: Pattern pattern = Pattern.compile(omitIndexingFilterConfigurationEntry.getRegex()); if (pattern.matcher(realUrl).matches()) { return null; } break; case META_FIELD_PRESENT: if(metadata.get(FilterUtils.getNullSafe(omitIndexingFilterConfigurationEntry.getName(), "")) != null) { return null; } break; } } return doc; } return null; } OmitIndexingFilter(); @Override Configuration getConf(); @Override void setConf(Configuration configuration); @Override NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks);  }
@Test public void testFilter() throws IndexingException { Parse parse = mock(Parse.class); Configuration configuration = mock(Configuration.class); ParseData parseData = new ParseData(); parseData.setParseMeta(metadata); when(parse.getData()).thenReturn(parseData); when(configuration.get(anyString())).thenReturn(""); when(configuration.getConfResourceAsReader(anyString())).thenReturn(new InputStreamReader(OmitIndexingFilterTest.class.getResourceAsStream("example-omit-indexfilter-conf.xml"))); omitIndexingFilter.setConf(configuration); NutchDocument nutchDocumentOut = omitIndexingFilter.filter(nutchDocumentIn, parse, new Text("http: assertNull("Document unsuccessfuly filtered!", nutchDocumentOut); nutchDocumentOut = omitIndexingFilter.filter(nutchDocumentIn, parse, new Text("http: assertSame("Document unsuccessfuly filtered!", nutchDocumentIn, nutchDocumentOut); }