@Test public void testNormalise() { final String[][] TESTS = new String[][] { {"http: {"http: {"http: {"http: {"http: {"http: {"http: }; Map<String, String> normMap = new HashMap<String, String>(); normMap.put(HtmlFeatureParser.CONF_LINKS_NORMALISE, "true"); HtmlFeatureParser normParser = new HtmlFeatureParser(ConfigFactory.parseMap(normMap)); normMap.put(HtmlFeatureParser.CONF_LINKS_NORMALISE, "false"); HtmlFeatureParser skipParser = new HtmlFeatureParser(ConfigFactory.parseMap(normMap)); for (String[] test: TESTS) { Assert.assertEquals("Normalisation of '" + test[1] + "'", test[0], normParser.normaliseLink(test[1])); Assert.assertEquals("Non-normalisation of '" + test[1] + "'", test[1], skipParser.normaliseLink(test[1])); } }
public String normaliseLink(String link) { return normaliseLinks ? Normalisation.canonicaliseURL(link) : link; }
HtmlFeatureParser extends AbstractParser { public String normaliseLink(String link) { return normaliseLinks ? Normalisation.canonicaliseURL(link) : link; } }
HtmlFeatureParser extends AbstractParser { public String normaliseLink(String link) { return normaliseLinks ? Normalisation.canonicaliseURL(link) : link; } HtmlFeatureParser();  HtmlFeatureParser(Config conf); }
HtmlFeatureParser extends AbstractParser { public String normaliseLink(String link) { return normaliseLinks ? Normalisation.canonicaliseURL(link) : link; } HtmlFeatureParser();  HtmlFeatureParser(Config conf); void setMaxParseErrors(int max_errors); int getMaxParseErrors(); List<ParseError> getParseErrors(); @Override Set<MediaType> getSupportedTypes(ParseContext context); @Override void parse(InputStream stream, ContentHandler handler,
            Metadata metadata, ParseContext context); String normaliseLink(String link); static Metadata extractMetadata( InputStream in, String url ); }
HtmlFeatureParser extends AbstractParser { public String normaliseLink(String link) { return normaliseLinks ? Normalisation.canonicaliseURL(link) : link; } HtmlFeatureParser();  HtmlFeatureParser(Config conf); void setMaxParseErrors(int max_errors); int getMaxParseErrors(); List<ParseError> getParseErrors(); @Override Set<MediaType> getSupportedTypes(ParseContext context); @Override void parse(InputStream stream, ContentHandler handler,
            Metadata metadata, ParseContext context); String normaliseLink(String link); static Metadata extractMetadata( InputStream in, String url ); static final String ORIGINAL_PUB_DATE; static final Property LINK_LIST; static final Property LINKS; static final String FIRST_PARAGRAPH; static final Property IMAGE_LINKS; static final Property DISTINCT_ELEMENTS; static final Property NUM_PARSE_ERRORS; static final int DEFAULT_MAX_PARSE_ERRORS; static final String CONF_LINKS_NORMALISE; static final String CONF_LINKS_EXTRACT_IMAGE_LINKS; static final boolean DEFAULT_LINKS_NORMALISE; }