@Test public void testFilter() throws IndexingException { NutchDocument nutchDocumentIn = new NutchDocument(); Parse parse = mock(Parse.class); ParseData parseData = new ParseData(); parseData.setParseMeta(metadata); Configuration configuration = mock(Configuration.class); when(parse.getData()).thenReturn(parseData); when(configuration.get(anyString())).thenReturn(""); when(configuration.getConfResourceAsReader(anyString())).thenReturn(new InputStreamReader(XPathIndexingFilterTest.class.getResourceAsStream("example-xpathfilter-conf.xml"))); xmlHtmlIndexingFilter.setConf(configuration); NutchDocument nutchDocumentOut = xmlHtmlIndexingFilter.filter(nutchDocumentIn, parse, new Text("www.test.com"), null, null); int stringValueIndexCount = 0; int floatValueIndexCount = 0; int dateValueIndexCount = 0; for(String fieldName : nutchDocumentOut.getFieldNames()) { for(Object value : nutchDocumentOut.getField(fieldName).getValues()) { if(fieldName.equals("testString")) { int index = Arrays.binarySearch(testStringArray, value); stringValueIndexCount += index; assertTrue(index >= 0); assertTrue(value instanceof String); } else if(fieldName.equals("testFloat")) { int index = Arrays.binarySearch(testFloatArray, value); floatValueIndexCount += index; assertTrue(index >= 0); assertTrue(value instanceof Float); } else if(fieldName.equals("testDate")) { int index = Arrays.binarySearch(testDateArray, value); dateValueIndexCount += index; assertTrue(index >= 0); assertTrue(value instanceof Date); } } } assertEquals("Not all String values parsed!", 3, stringValueIndexCount); assertEquals("Not all Flaot values parsed!", 1, floatValueIndexCount); assertEquals("Not all Flaot values parsed!", 1, dateValueIndexCount); }
@Override public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { Metadata metadata = parse.getData().getParseMeta(); List<XPathIndexerProperties> xPathIndexerPropertiesList = xpathFilterConfiguration.getXPathIndexerPropertiesList(); for(XPathIndexerProperties xPathIndexerProperties : xPathIndexerPropertiesList) { if(FilterUtils.isMatch(xPathIndexerProperties.getPageUrlFilterRegex(), new String(url.getBytes()).substring(0, url.getLength()))) { List<XPathIndexerPropertiesField> xPathIndexerPropertiesFieldList = xPathIndexerProperties.getXPathIndexerPropertiesFieldList(); for(XPathIndexerPropertiesField xPathIndexerPropertiesField : xPathIndexerPropertiesFieldList) { FieldType type = xPathIndexerPropertiesField.getType(); for(String stringValue : metadata.getValues(xPathIndexerPropertiesField.getName())) { Object value; switch(type) { case STRING: value = stringValue; break; case INTEGER: value = Integer.valueOf(stringValue); break; case LONG: value = Long.valueOf(stringValue); break; case DOUBLE: value = Double.valueOf(stringValue); break; case FLOAT: value = Float.valueOf(stringValue); break; case BOOLEAN: value = Boolean.valueOf(stringValue); break; case DATE: String dateFormat = xPathIndexerPropertiesField.getDateFormat() == null?"dd.MM.yyyy":xPathIndexerPropertiesField.getDateFormat(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat(dateFormat); try { value = simpleDateFormat.parseObject(stringValue); } catch (ParseException e) { value = new Date(0); } break; default: log.warn(String.format("Type '%s' not supported, value will be interpreted as String", type)); value = stringValue; break; } doc.add(xPathIndexerPropertiesField.getName(), value); if (log.isDebugEnabled()) { log.debug(String.format("Added field with name %s and value %s", xPathIndexerPropertiesField.getName(), value)); } } } } } return doc; }
XPathIndexingFilter implements IndexingFilter { @Override public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { Metadata metadata = parse.getData().getParseMeta(); List<XPathIndexerProperties> xPathIndexerPropertiesList = xpathFilterConfiguration.getXPathIndexerPropertiesList(); for(XPathIndexerProperties xPathIndexerProperties : xPathIndexerPropertiesList) { if(FilterUtils.isMatch(xPathIndexerProperties.getPageUrlFilterRegex(), new String(url.getBytes()).substring(0, url.getLength()))) { List<XPathIndexerPropertiesField> xPathIndexerPropertiesFieldList = xPathIndexerProperties.getXPathIndexerPropertiesFieldList(); for(XPathIndexerPropertiesField xPathIndexerPropertiesField : xPathIndexerPropertiesFieldList) { FieldType type = xPathIndexerPropertiesField.getType(); for(String stringValue : metadata.getValues(xPathIndexerPropertiesField.getName())) { Object value; switch(type) { case STRING: value = stringValue; break; case INTEGER: value = Integer.valueOf(stringValue); break; case LONG: value = Long.valueOf(stringValue); break; case DOUBLE: value = Double.valueOf(stringValue); break; case FLOAT: value = Float.valueOf(stringValue); break; case BOOLEAN: value = Boolean.valueOf(stringValue); break; case DATE: String dateFormat = xPathIndexerPropertiesField.getDateFormat() == null?"dd.MM.yyyy":xPathIndexerPropertiesField.getDateFormat(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat(dateFormat); try { value = simpleDateFormat.parseObject(stringValue); } catch (ParseException e) { value = new Date(0); } break; default: log.warn(String.format("Type '%s' not supported, value will be interpreted as String", type)); value = stringValue; break; } doc.add(xPathIndexerPropertiesField.getName(), value); if (log.isDebugEnabled()) { log.debug(String.format("Added field with name %s and value %s", xPathIndexerPropertiesField.getName(), value)); } } } } } return doc; } }
XPathIndexingFilter implements IndexingFilter { @Override public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { Metadata metadata = parse.getData().getParseMeta(); List<XPathIndexerProperties> xPathIndexerPropertiesList = xpathFilterConfiguration.getXPathIndexerPropertiesList(); for(XPathIndexerProperties xPathIndexerProperties : xPathIndexerPropertiesList) { if(FilterUtils.isMatch(xPathIndexerProperties.getPageUrlFilterRegex(), new String(url.getBytes()).substring(0, url.getLength()))) { List<XPathIndexerPropertiesField> xPathIndexerPropertiesFieldList = xPathIndexerProperties.getXPathIndexerPropertiesFieldList(); for(XPathIndexerPropertiesField xPathIndexerPropertiesField : xPathIndexerPropertiesFieldList) { FieldType type = xPathIndexerPropertiesField.getType(); for(String stringValue : metadata.getValues(xPathIndexerPropertiesField.getName())) { Object value; switch(type) { case STRING: value = stringValue; break; case INTEGER: value = Integer.valueOf(stringValue); break; case LONG: value = Long.valueOf(stringValue); break; case DOUBLE: value = Double.valueOf(stringValue); break; case FLOAT: value = Float.valueOf(stringValue); break; case BOOLEAN: value = Boolean.valueOf(stringValue); break; case DATE: String dateFormat = xPathIndexerPropertiesField.getDateFormat() == null?"dd.MM.yyyy":xPathIndexerPropertiesField.getDateFormat(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat(dateFormat); try { value = simpleDateFormat.parseObject(stringValue); } catch (ParseException e) { value = new Date(0); } break; default: log.warn(String.format("Type '%s' not supported, value will be interpreted as String", type)); value = stringValue; break; } doc.add(xPathIndexerPropertiesField.getName(), value); if (log.isDebugEnabled()) { log.debug(String.format("Added field with name %s and value %s", xPathIndexerPropertiesField.getName(), value)); } } } } } return doc; } XPathIndexingFilter(); }
XPathIndexingFilter implements IndexingFilter { @Override public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { Metadata metadata = parse.getData().getParseMeta(); List<XPathIndexerProperties> xPathIndexerPropertiesList = xpathFilterConfiguration.getXPathIndexerPropertiesList(); for(XPathIndexerProperties xPathIndexerProperties : xPathIndexerPropertiesList) { if(FilterUtils.isMatch(xPathIndexerProperties.getPageUrlFilterRegex(), new String(url.getBytes()).substring(0, url.getLength()))) { List<XPathIndexerPropertiesField> xPathIndexerPropertiesFieldList = xPathIndexerProperties.getXPathIndexerPropertiesFieldList(); for(XPathIndexerPropertiesField xPathIndexerPropertiesField : xPathIndexerPropertiesFieldList) { FieldType type = xPathIndexerPropertiesField.getType(); for(String stringValue : metadata.getValues(xPathIndexerPropertiesField.getName())) { Object value; switch(type) { case STRING: value = stringValue; break; case INTEGER: value = Integer.valueOf(stringValue); break; case LONG: value = Long.valueOf(stringValue); break; case DOUBLE: value = Double.valueOf(stringValue); break; case FLOAT: value = Float.valueOf(stringValue); break; case BOOLEAN: value = Boolean.valueOf(stringValue); break; case DATE: String dateFormat = xPathIndexerPropertiesField.getDateFormat() == null?"dd.MM.yyyy":xPathIndexerPropertiesField.getDateFormat(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat(dateFormat); try { value = simpleDateFormat.parseObject(stringValue); } catch (ParseException e) { value = new Date(0); } break; default: log.warn(String.format("Type '%s' not supported, value will be interpreted as String", type)); value = stringValue; break; } doc.add(xPathIndexerPropertiesField.getName(), value); if (log.isDebugEnabled()) { log.debug(String.format("Added field with name %s and value %s", xPathIndexerPropertiesField.getName(), value)); } } } } } return doc; } XPathIndexingFilter(); @Override Configuration getConf(); @Override void setConf(Configuration configuration); @Override NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks); }
XPathIndexingFilter implements IndexingFilter { @Override public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { Metadata metadata = parse.getData().getParseMeta(); List<XPathIndexerProperties> xPathIndexerPropertiesList = xpathFilterConfiguration.getXPathIndexerPropertiesList(); for(XPathIndexerProperties xPathIndexerProperties : xPathIndexerPropertiesList) { if(FilterUtils.isMatch(xPathIndexerProperties.getPageUrlFilterRegex(), new String(url.getBytes()).substring(0, url.getLength()))) { List<XPathIndexerPropertiesField> xPathIndexerPropertiesFieldList = xPathIndexerProperties.getXPathIndexerPropertiesFieldList(); for(XPathIndexerPropertiesField xPathIndexerPropertiesField : xPathIndexerPropertiesFieldList) { FieldType type = xPathIndexerPropertiesField.getType(); for(String stringValue : metadata.getValues(xPathIndexerPropertiesField.getName())) { Object value; switch(type) { case STRING: value = stringValue; break; case INTEGER: value = Integer.valueOf(stringValue); break; case LONG: value = Long.valueOf(stringValue); break; case DOUBLE: value = Double.valueOf(stringValue); break; case FLOAT: value = Float.valueOf(stringValue); break; case BOOLEAN: value = Boolean.valueOf(stringValue); break; case DATE: String dateFormat = xPathIndexerPropertiesField.getDateFormat() == null?"dd.MM.yyyy":xPathIndexerPropertiesField.getDateFormat(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat(dateFormat); try { value = simpleDateFormat.parseObject(stringValue); } catch (ParseException e) { value = new Date(0); } break; default: log.warn(String.format("Type '%s' not supported, value will be interpreted as String", type)); value = stringValue; break; } doc.add(xPathIndexerPropertiesField.getName(), value); if (log.isDebugEnabled()) { log.debug(String.format("Added field with name %s and value %s", xPathIndexerPropertiesField.getName(), value)); } } } } } return doc; } XPathIndexingFilter(); @Override Configuration getConf(); @Override void setConf(Configuration configuration); @Override NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks);  }