@Test public void JSoupElementsAddedTest() throws Exception { String testHtml = "<b>test<p></b><script> if (3<a) console.log('something');</script>"; String baseUri = "http: ByteArrayInputStream stream = new ByteArrayInputStream( testHtml.getBytes()); stream.mark(100000); Parser parser = Parser.xmlParser(); parser.setTrackErrors(1000); Document doc = Jsoup.parse(stream, null, baseUri, parser); System.out.println("Parsed as:\n" + doc); System.out.println("Got " + parser.getErrors().size() + " under isTrackErrors = " + parser.isTrackErrors()); for (ParseError err : parser.getErrors()) { System.out.println("Got: " + err); } stream.reset(); innerBasicParseTest(stream, baseUri, 3); }
@Override public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { final long start = System.nanoTime(); String url = metadata.get( Metadata.RESOURCE_NAME_KEY ); Document doc = null; try { doc = Jsoup.parse(stream, null, url, parser); } catch (java.nio.charset.IllegalCharsetNameException e ) { log.warn("Jsoup parse had to assume UTF-8: "+e); doc = Jsoup.parse(stream, "UTF-8", url ); } catch( Exception e ) { log.error("Jsoup parse failed: "+e); } finally { if( doc == null ) return; } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#jsoupparse", start); final long nonJsoupStart = System.nanoTime(); if (parser.getErrors() != null) metadata.set(NUM_PARSE_ERRORS, parser.getErrors().size()); Set<String> links = this.extractLinks(doc); if( links != null && links.size() > 0 ) { metadata.set(LINK_LIST, links.toArray(new String[links.size()])); } if (extractImageLinks){ Set<String> imageLinks = this.extractImageLinks(doc); if( imageLinks != null && imageLinks .size() > 0 ) { metadata.set(IMAGE_LINKS, imageLinks.toArray(new String[imageLinks.size()])); } } for( Element meta : doc.select("meta[name=OriginalPublicationDate]") ) { metadata.set(ORIGINAL_PUB_DATE, meta.attr("content")); } for( Element p : doc.select("p") ) { String pt = p.text(); if( pt != null ) { pt = pt.trim(); if( pt.length() > 0 ) { metadata.set(FIRST_PARAGRAPH, p.text() ); break; } } } Set<String> de = new HashSet<String>(); for( Element e : doc.select("*") ) { if( !"#root".equals(e.tag().getName()) && ELEMENT_NAME.matcher(e.tag().getName()).matches() ) { de.add(StringUtils.left(e.tag().getName().toLowerCase(Locale.ENGLISH), 100)); } } for (Element e : doc.select("link")) { if (e.attr("rel") != null) { de.add("link/@rel=" + e.attr("rel").toLowerCase()); } } for (Element e : doc.select("meta")) { if (e.attr("name") != null) { de.add("meta/@name=" + e.attr("name").toLowerCase()); } } metadata.set(DISTINCT_ELEMENTS, de.toArray(new String[] {})); for( Element a : doc.select("a[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("link[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("area[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#featureextract", nonJsoupStart); }
HtmlFeatureParser extends AbstractParser { @Override public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { final long start = System.nanoTime(); String url = metadata.get( Metadata.RESOURCE_NAME_KEY ); Document doc = null; try { doc = Jsoup.parse(stream, null, url, parser); } catch (java.nio.charset.IllegalCharsetNameException e ) { log.warn("Jsoup parse had to assume UTF-8: "+e); doc = Jsoup.parse(stream, "UTF-8", url ); } catch( Exception e ) { log.error("Jsoup parse failed: "+e); } finally { if( doc == null ) return; } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#jsoupparse", start); final long nonJsoupStart = System.nanoTime(); if (parser.getErrors() != null) metadata.set(NUM_PARSE_ERRORS, parser.getErrors().size()); Set<String> links = this.extractLinks(doc); if( links != null && links.size() > 0 ) { metadata.set(LINK_LIST, links.toArray(new String[links.size()])); } if (extractImageLinks){ Set<String> imageLinks = this.extractImageLinks(doc); if( imageLinks != null && imageLinks .size() > 0 ) { metadata.set(IMAGE_LINKS, imageLinks.toArray(new String[imageLinks.size()])); } } for( Element meta : doc.select("meta[name=OriginalPublicationDate]") ) { metadata.set(ORIGINAL_PUB_DATE, meta.attr("content")); } for( Element p : doc.select("p") ) { String pt = p.text(); if( pt != null ) { pt = pt.trim(); if( pt.length() > 0 ) { metadata.set(FIRST_PARAGRAPH, p.text() ); break; } } } Set<String> de = new HashSet<String>(); for( Element e : doc.select("*") ) { if( !"#root".equals(e.tag().getName()) && ELEMENT_NAME.matcher(e.tag().getName()).matches() ) { de.add(StringUtils.left(e.tag().getName().toLowerCase(Locale.ENGLISH), 100)); } } for (Element e : doc.select("link")) { if (e.attr("rel") != null) { de.add("link/@rel=" + e.attr("rel").toLowerCase()); } } for (Element e : doc.select("meta")) { if (e.attr("name") != null) { de.add("meta/@name=" + e.attr("name").toLowerCase()); } } metadata.set(DISTINCT_ELEMENTS, de.toArray(new String[] {})); for( Element a : doc.select("a[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("link[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("area[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#featureextract", nonJsoupStart); } }
HtmlFeatureParser extends AbstractParser { @Override public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { final long start = System.nanoTime(); String url = metadata.get( Metadata.RESOURCE_NAME_KEY ); Document doc = null; try { doc = Jsoup.parse(stream, null, url, parser); } catch (java.nio.charset.IllegalCharsetNameException e ) { log.warn("Jsoup parse had to assume UTF-8: "+e); doc = Jsoup.parse(stream, "UTF-8", url ); } catch( Exception e ) { log.error("Jsoup parse failed: "+e); } finally { if( doc == null ) return; } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#jsoupparse", start); final long nonJsoupStart = System.nanoTime(); if (parser.getErrors() != null) metadata.set(NUM_PARSE_ERRORS, parser.getErrors().size()); Set<String> links = this.extractLinks(doc); if( links != null && links.size() > 0 ) { metadata.set(LINK_LIST, links.toArray(new String[links.size()])); } if (extractImageLinks){ Set<String> imageLinks = this.extractImageLinks(doc); if( imageLinks != null && imageLinks .size() > 0 ) { metadata.set(IMAGE_LINKS, imageLinks.toArray(new String[imageLinks.size()])); } } for( Element meta : doc.select("meta[name=OriginalPublicationDate]") ) { metadata.set(ORIGINAL_PUB_DATE, meta.attr("content")); } for( Element p : doc.select("p") ) { String pt = p.text(); if( pt != null ) { pt = pt.trim(); if( pt.length() > 0 ) { metadata.set(FIRST_PARAGRAPH, p.text() ); break; } } } Set<String> de = new HashSet<String>(); for( Element e : doc.select("*") ) { if( !"#root".equals(e.tag().getName()) && ELEMENT_NAME.matcher(e.tag().getName()).matches() ) { de.add(StringUtils.left(e.tag().getName().toLowerCase(Locale.ENGLISH), 100)); } } for (Element e : doc.select("link")) { if (e.attr("rel") != null) { de.add("link/@rel=" + e.attr("rel").toLowerCase()); } } for (Element e : doc.select("meta")) { if (e.attr("name") != null) { de.add("meta/@name=" + e.attr("name").toLowerCase()); } } metadata.set(DISTINCT_ELEMENTS, de.toArray(new String[] {})); for( Element a : doc.select("a[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("link[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("area[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#featureextract", nonJsoupStart); } HtmlFeatureParser();  HtmlFeatureParser(Config conf); }
HtmlFeatureParser extends AbstractParser { @Override public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { final long start = System.nanoTime(); String url = metadata.get( Metadata.RESOURCE_NAME_KEY ); Document doc = null; try { doc = Jsoup.parse(stream, null, url, parser); } catch (java.nio.charset.IllegalCharsetNameException e ) { log.warn("Jsoup parse had to assume UTF-8: "+e); doc = Jsoup.parse(stream, "UTF-8", url ); } catch( Exception e ) { log.error("Jsoup parse failed: "+e); } finally { if( doc == null ) return; } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#jsoupparse", start); final long nonJsoupStart = System.nanoTime(); if (parser.getErrors() != null) metadata.set(NUM_PARSE_ERRORS, parser.getErrors().size()); Set<String> links = this.extractLinks(doc); if( links != null && links.size() > 0 ) { metadata.set(LINK_LIST, links.toArray(new String[links.size()])); } if (extractImageLinks){ Set<String> imageLinks = this.extractImageLinks(doc); if( imageLinks != null && imageLinks .size() > 0 ) { metadata.set(IMAGE_LINKS, imageLinks.toArray(new String[imageLinks.size()])); } } for( Element meta : doc.select("meta[name=OriginalPublicationDate]") ) { metadata.set(ORIGINAL_PUB_DATE, meta.attr("content")); } for( Element p : doc.select("p") ) { String pt = p.text(); if( pt != null ) { pt = pt.trim(); if( pt.length() > 0 ) { metadata.set(FIRST_PARAGRAPH, p.text() ); break; } } } Set<String> de = new HashSet<String>(); for( Element e : doc.select("*") ) { if( !"#root".equals(e.tag().getName()) && ELEMENT_NAME.matcher(e.tag().getName()).matches() ) { de.add(StringUtils.left(e.tag().getName().toLowerCase(Locale.ENGLISH), 100)); } } for (Element e : doc.select("link")) { if (e.attr("rel") != null) { de.add("link/@rel=" + e.attr("rel").toLowerCase()); } } for (Element e : doc.select("meta")) { if (e.attr("name") != null) { de.add("meta/@name=" + e.attr("name").toLowerCase()); } } metadata.set(DISTINCT_ELEMENTS, de.toArray(new String[] {})); for( Element a : doc.select("a[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("link[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("area[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#featureextract", nonJsoupStart); } HtmlFeatureParser();  HtmlFeatureParser(Config conf); void setMaxParseErrors(int max_errors); int getMaxParseErrors(); List<ParseError> getParseErrors(); @Override Set<MediaType> getSupportedTypes(ParseContext context); @Override void parse(InputStream stream, ContentHandler handler,
            Metadata metadata, ParseContext context); String normaliseLink(String link); static Metadata extractMetadata( InputStream in, String url ); }
HtmlFeatureParser extends AbstractParser { @Override public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { final long start = System.nanoTime(); String url = metadata.get( Metadata.RESOURCE_NAME_KEY ); Document doc = null; try { doc = Jsoup.parse(stream, null, url, parser); } catch (java.nio.charset.IllegalCharsetNameException e ) { log.warn("Jsoup parse had to assume UTF-8: "+e); doc = Jsoup.parse(stream, "UTF-8", url ); } catch( Exception e ) { log.error("Jsoup parse failed: "+e); } finally { if( doc == null ) return; } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#jsoupparse", start); final long nonJsoupStart = System.nanoTime(); if (parser.getErrors() != null) metadata.set(NUM_PARSE_ERRORS, parser.getErrors().size()); Set<String> links = this.extractLinks(doc); if( links != null && links.size() > 0 ) { metadata.set(LINK_LIST, links.toArray(new String[links.size()])); } if (extractImageLinks){ Set<String> imageLinks = this.extractImageLinks(doc); if( imageLinks != null && imageLinks .size() > 0 ) { metadata.set(IMAGE_LINKS, imageLinks.toArray(new String[imageLinks.size()])); } } for( Element meta : doc.select("meta[name=OriginalPublicationDate]") ) { metadata.set(ORIGINAL_PUB_DATE, meta.attr("content")); } for( Element p : doc.select("p") ) { String pt = p.text(); if( pt != null ) { pt = pt.trim(); if( pt.length() > 0 ) { metadata.set(FIRST_PARAGRAPH, p.text() ); break; } } } Set<String> de = new HashSet<String>(); for( Element e : doc.select("*") ) { if( !"#root".equals(e.tag().getName()) && ELEMENT_NAME.matcher(e.tag().getName()).matches() ) { de.add(StringUtils.left(e.tag().getName().toLowerCase(Locale.ENGLISH), 100)); } } for (Element e : doc.select("link")) { if (e.attr("rel") != null) { de.add("link/@rel=" + e.attr("rel").toLowerCase()); } } for (Element e : doc.select("meta")) { if (e.attr("name") != null) { de.add("meta/@name=" + e.attr("name").toLowerCase()); } } metadata.set(DISTINCT_ELEMENTS, de.toArray(new String[] {})); for( Element a : doc.select("a[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("link[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } for( Element a : doc.select("area[rel=license]") ) { metadata.add( Metadata.LICENSE_URL, a.attr("href") ); } Instrument.timeRel("HTMLAnalyzer.analyze#parser", "HtmlFeatureParser.parse#featureextract", nonJsoupStart); } HtmlFeatureParser();  HtmlFeatureParser(Config conf); void setMaxParseErrors(int max_errors); int getMaxParseErrors(); List<ParseError> getParseErrors(); @Override Set<MediaType> getSupportedTypes(ParseContext context); @Override void parse(InputStream stream, ContentHandler handler,
            Metadata metadata, ParseContext context); String normaliseLink(String link); static Metadata extractMetadata( InputStream in, String url ); static final String ORIGINAL_PUB_DATE; static final Property LINK_LIST; static final Property LINKS; static final String FIRST_PARAGRAPH; static final Property IMAGE_LINKS; static final Property DISTINCT_ELEMENTS; static final Property NUM_PARSE_ERRORS; static final int DEFAULT_MAX_PARSE_ERRORS; static final String CONF_LINKS_NORMALISE; static final String CONF_LINKS_EXTRACT_IMAGE_LINKS; static final boolean DEFAULT_LINKS_NORMALISE; }