HtmlParser extends AbstractEncodingDetectorParser { public void parse( InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { TemporaryResources tmp = null; try { if (!TikaInputStream.isTikaInputStream(stream)) { tmp = new TemporaryResources(); stream = TikaInputStream.get(stream, tmp); } parseImpl(stream, handler, metadata, context); } finally { if (tmp != null) { tmp.close(); } } } HtmlParser();  HtmlParser(EncodingDetector encodingDetector); Set<MediaType> getSupportedTypes(ParseContext context); void parse(
            InputStream stream, ContentHandler handler,
            Metadata metadata, ParseContext context); String mapSafeAttribute(String elementName, String attributeName); @Field void setExtractScripts(boolean extractScripts); boolean getExtractScripts();  }
@Test public void testHttpEquivCharset() throws Exception { String test = "<html><head><meta http-equiv=\"content-type\"" + " content=\"text/html; charset=ISO-8859-1\" />" + "<title>the name is \u00e1ndre</title>" + "</head><body></body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("ISO-8859-1", metadata.get(Metadata.CONTENT_ENCODING)); }
@Test public void testHtml5Charset() throws Exception { String test = "<html><head><meta charset=\"ISO-8859-15\" />" + "<title>the name is \u00e1ndre</title>" + "</head><body></body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("ISO-8859-15", metadata.get(Metadata.CONTENT_ENCODING)); }
@Test public void testDetectOfCharset() throws Exception { String test = "<html><head><title>\u017d</title></head><body></body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("\u017d", metadata.get(TikaCoreProperties.TITLE)); }
@Test public void testUsingCharsetInContentTypeHeader() throws Exception { final String test = "<html><head><title>the name is \u00e1ndre</title></head>" + "<body></body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("UTF-8", metadata.get(Metadata.CONTENT_ENCODING)); metadata = new Metadata(); metadata.set(Metadata.CONTENT_TYPE, "text/html; charset=ISO-8859-1"); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("ISO-8859-1", metadata.get(Metadata.CONTENT_ENCODING)); }
@Test public void testIgnoreCharsetDetectorLanguage() throws Exception { String test = "<html><title>Simple Content</title><body></body></html>"; Metadata metadata = new Metadata(); metadata.add(Metadata.CONTENT_LANGUAGE, "en"); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("en", metadata.get(Metadata.CONTENT_LANGUAGE)); }
@Test public void testHttpEquivCharsetFunkyAttributes() throws Exception { String test1 = "<html><head><meta http-equiv=\"content-type\"" + " content=\"text/html; charset=ISO-8859-15; charset=iso-8859-15\" />" + "<title>the name is \u00e1ndre</title>" + "</head><body></body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test1.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("ISO-8859-15", metadata.get(Metadata.CONTENT_ENCODING)); String test2 = "<html><head><meta http-equiv=\"content-type\"" + " content=\"text/html;;charset=ISO-8859-15\" />" + "<title>the name is \u00e1ndre</title>" + "</head><body></body></html>"; metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test2.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("ISO-8859-15", metadata.get(Metadata.CONTENT_ENCODING)); }
@Test public void testUsingFunkyCharsetInContentTypeHeader() throws Exception { final String test = "<html><head><title>the name is \u00e1ndre</title></head>" + "<body></body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("UTF-8", metadata.get(Metadata.CONTENT_ENCODING)); metadata = new Metadata(); metadata.set(Metadata.CONTENT_TYPE, "charset=ISO-8859-1;text/html"); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("ISO-8859-1", metadata.get(Metadata.CONTENT_ENCODING)); }
@Test public void testMetaHttpEquivWithLotsOfPreambleText() throws Exception { String path = "/test-documents/big-preamble.html"; Metadata metadata = new Metadata(); new HtmlParser().parse( HtmlParserTest.class.getResourceAsStream(path), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("windows-1251", metadata.get(Metadata.CONTENT_ENCODING)); }
@Test public void testBoilerplateRemoval() throws Exception { String path = "/test-documents/boilerplate.html"; Metadata metadata = new Metadata(); BodyContentHandler handler = new BodyContentHandler(); new HtmlParser().parse( HtmlParserTest.class.getResourceAsStream(path), new BoilerpipeContentHandler(handler), metadata, new ParseContext()); String content = handler.toString(); assertTrue(content.startsWith("This is the real meat")); assertTrue(content.endsWith("This is the end of the text.\n")); assertFalse(content.contains("boilerplate")); assertFalse(content.contains("footer")); }
@Test public void testElementOrdering() throws Exception { final String test = "<html><head><title>Title</title>" + "<meta http-equiv=\"content-type\" content=\"text/html\">" + "<link rel=\"next\" href=\"next.html\" />" + "</head><body><p>Simple Content</p></body></html>"; StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), makeHtmlTransformer(sw), new Metadata(), new ParseContext()); String result = sw.toString(); assertTrue(Pattern.matches("(?s)<html.*<head>.*<title>Title</title>.*</head>.*$", result)); assertFalse(Pattern.matches("(?s).*<body>.*<meta. *</body>.*$", result)); assertTrue(Pattern.matches("(?s)<html.*<head>.*<meta .*</head>.*$", result)); assertFalse(Pattern.matches("(?s).*<body>.*<link .*</body>.*$", result)); assertTrue(Pattern.matches("(?s)<html.*<head>.*<link .*</head>.*$", result)); assertTrue(Pattern.matches("(?s).*</body>.*</html>$", result)); }
@Test public void testImgUrlExtraction() throws Exception { final String test = "<html><head><title>Title</title>" + "<base href=\"http: "</head><body><img src=\"image.jpg\" /></body></html>"; StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), makeHtmlTransformer(sw), new Metadata(), new ParseContext()); String result = sw.toString(); assertTrue(Pattern.matches("(?s).*src=\"http: }
@Test public void testFrameSrcExtraction() throws Exception { final String test = "<html><head><title>Title</title>" + "<base href=\"http: "</head><frameset><frame src=\"frame.html\" /></frameset></html>"; StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), makeHtmlTransformer(sw), new Metadata(), new ParseContext()); String result = sw.toString(); assertTrue(Pattern.matches("(?s).*<frame .* src=\"http: }
@Test public void testIFrameSrcExtraction() throws Exception { final String test = "<html><head><title>Title</title>" + "<base href=\"http: "</head><body><iframe src =\"framed.html\" width=\"100%\" height=\"300\">" + "<p>Your browser doesn't support iframes!</p></body></html>"; StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), makeHtmlTransformer(sw), new Metadata(), new ParseContext()); String result = sw.toString(); assertTrue(Pattern.matches("(?s).*<iframe .* src=\"http: }
@Test public void testAreaExtraction() throws Exception { final String test = "<html><head><title>Title</title>" + "<base href=\"http: "</head><body><p><map name=\"map\" id=\"map\">" + "<area shape=\"rect\" href=\"map.html\" alt=\"\" />" + "</map></p></body></html>"; StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), makeHtmlTransformer(sw), new Metadata(), new ParseContext()); String result = sw.toString(); assertTrue(Pattern.matches("(?s).*<map .*<area .* href=\"http: }
@Test public void testObjectExtraction() throws Exception { final String test = "<html><head><title>Title</title>" + "<base href=\"http: "</head><body><p><object data=\"object.data\" type=\"text/html\">" + "<param name=\"name\" value=\"value\" />" + "</object></p></body></html>"; StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), makeHtmlTransformer(sw), new Metadata(), new ParseContext()); String result = sw.toString(); assertTrue( "<object> tag not correctly found in:\n" + result, Pattern.matches("(?s).*<object data=\"http: ); }
@Test public void testMetaTagHandling() throws Exception { final String test = "<html><body><h1>header</h1><p>some text</p></body></html>"; Metadata metadata = new Metadata(); metadata.add("Content-Type", "text/html; charset=utf-8"); metadata.add("Language", null); StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), makeHtmlTransformer(sw), metadata, new ParseContext()); String result = sw.toString(); assertTrue(Pattern.matches("(?s).*<meta name=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>.*$", result)); assertFalse(Pattern.matches("(?s).*<meta name=\"Language\".*$", result)); }
@Test public void testBrokenFrameset() throws Exception { final String test1 = "<html><head><title>Title</title>" + "<base href=\"http: "</head><body><frameset><frame src=\"frame.html\" /></frameset></body></html>"; StringWriter sw1 = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test1.getBytes(UTF_8)), makeHtmlTransformer(sw1), new Metadata(), new ParseContext()); String result = sw1.toString(); assertTrue(Pattern.matches("(?s).*<frame .* src=\"http: assertFalse(Pattern.matches("(?s).*<body>.*$", result)); final String test2 = "<html><head><title> my title </title></head><body>" + "<frameset rows=\"20,*\"><frame src=\"top.html\"></frame>" + "<frameset cols=\"20,*\"><frame src=\"left.html\"></frame>" + "<frame src=\"invalid.html\"/></frame>" + "<frame src=\"right.html\"></frame>" + "</frameset></frameset></body></html>"; StringWriter sw2 = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test2.getBytes(UTF_8)), makeHtmlTransformer(sw2), new Metadata(), new ParseContext()); result = sw2.toString(); assertTrue(Pattern.matches("(?s).*<frame .* src=\"top.html\"/>.*$", result)); assertTrue(Pattern.matches("(?s).*<frame .* src=\"left.html\"/>.*$", result)); assertTrue(Pattern.matches("(?s).*<frame .* src=\"invalid.html\"/>.*$", result)); assertTrue(Pattern.matches("(?s).*<frame .* src=\"right.html\"/>.*$", result)); assertFalse(Pattern.matches("(?s).*<body>.*$", result)); }
@Test public void testBoilerplateDelegation() throws Exception { String path = "/test-documents/boilerplate.html"; Metadata metadata = new Metadata(); StringWriter sw = new StringWriter(); new HtmlParser().parse( HtmlParserTest.class.getResourceAsStream(path), makeHtmlTransformer(sw), metadata, new ParseContext()); String content = sw.toString(); assertTrue(Pattern.matches("(?s).*<html xmlns=\"http: assertTrue(Pattern.matches("(?s).*<head>.*</head>.*$", content)); assertTrue(Pattern.matches("(?s).*<title>Title</title>.*$", content)); assertTrue(Pattern.matches("(?s).*<body>.*</body>.*$", content)); }
@Test public void testLinkHrefResolution() throws Exception { final String test = "<html><head><title>Title</title>" + "<base href=\"http: "<link rel=\"next\" href=\"next.html\" />" + "</head><body></body></html>"; StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), makeHtmlTransformer(sw), new Metadata(), new ParseContext()); String result = sw.toString(); assertTrue(Pattern.matches("(?s).*<head>.*<link rel=\"next\" href=\"http: }
@Test public void testBoilerplateWithMarkup() throws Exception { String path = "/test-documents/boilerplate.html"; Metadata metadata = new Metadata(); StringWriter sw = new StringWriter(); ContentHandler ch = makeHtmlTransformer(sw); BoilerpipeContentHandler bpch = new BoilerpipeContentHandler(ch); bpch.setIncludeMarkup(true); new HtmlParser().parse( HtmlParserTest.class.getResourceAsStream(path), bpch, metadata, new ParseContext()); String content = sw.toString(); assertTrue("Has empty table elements", content.contains("<body><table><tr><td><table><tr><td>")); assertTrue("Has empty a element", content.contains("<a shape=\"rect\" href=\"Main.php\"/>")); assertTrue("Has real content", content.contains("<p>This is the real meat")); assertTrue("Ends with appropriate HTML", content.endsWith("</p></body></html>")); assertFalse(content.contains("boilerplate")); assertFalse(content.contains("footer")); }
@Test public void testIdentityMapper() throws Exception { final String html = "<html><head><title>Title</title></head>" + "<body></body></html>"; Metadata metadata = new Metadata(); ParseContext parseContext = new ParseContext(); parseContext.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE); StringWriter sw = new StringWriter(); new HtmlParser().parse( new ByteArrayInputStream(html.getBytes(UTF_8)), makeHtmlTransformer(sw), metadata, parseContext); String result = sw.toString(); assertTrue(Pattern.matches("(?s).*<body/>.*$", result)); }
@Test public void testNewlineAndIndent() throws Exception { final String html = "<html><head><title>Title</title></head>" + "<body><ul><li>one</li></ul></body></html>"; BodyContentHandler handler = new BodyContentHandler(); new HtmlParser().parse( new ByteArrayInputStream(html.getBytes(UTF_8)), handler, new Metadata(), new ParseContext()); String result = handler.toString(); assertTrue(Pattern.matches("\tone\n\n", result)); }
@Test public void testHtmlLanguage() throws Exception { final String html = "<html lang=\"fr\"></html>"; StringWriter sw = new StringWriter(); Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(html.getBytes(UTF_8)), makeHtmlTransformer(sw), metadata, new ParseContext()); assertEquals("fr", metadata.get(Metadata.CONTENT_LANGUAGE)); assertTrue("Missing HTML lang attribute", Pattern.matches("(?s)<html[^>]* lang=\"fr\".*", sw.toString())); }
@Test public void testBoilerplateWhitespace() throws Exception { String path = "/test-documents/boilerplate-whitespace.html"; Metadata metadata = new Metadata(); BodyContentHandler handler = new BodyContentHandler(); BoilerpipeContentHandler bpHandler = new BoilerpipeContentHandler(handler); bpHandler.setIncludeMarkup(true); new HtmlParser().parse( HtmlParserTest.class.getResourceAsStream(path), bpHandler, metadata, new ParseContext()); String content = handler.toString(); assertFalse(content.contains("item_aitem_b")); assertContains("item_a\nitem_b", content); assertContains("有什么需要我帮你的", content); }
@Test public void testBoilerplateMissingWhitespace() throws Exception { String path = "/test-documents/testBoilerplateMissingSpace.html"; Metadata metadata = new Metadata(); BodyContentHandler handler = new BodyContentHandler(); BoilerpipeContentHandler bpHandler = new BoilerpipeContentHandler(handler); bpHandler.setIncludeMarkup(true); new HtmlParser().parse( HtmlParserTest.class.getResourceAsStream(path), bpHandler, metadata, new ParseContext()); String content = handler.toString(); assertContains("family Psychrolutidae", content); assertContains("(Psychrolutes marcidus)", content); }
@Test public void testOpenGraphMetadata() throws Exception { String test1 = "<html><head><meta property=\"og:description\"" + " content=\"some description\" />" + "<meta property=\"og:image\" content=\"http: + "<meta property=\"og:image\" content=\"http: + "<title>hello</title>" + "</head><body></body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test1.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("some description", metadata.get("og:description")); assertTrue(metadata.isMultiValued("og:image")); }
@Test public void testCustomHtmlSchema() throws Exception { String test = "<html><body><a><table><tr><td>text</tr></tr></table></a></body></html>"; Metadata metadata = new Metadata(); LinkContentHandler linkContentHandler = new LinkContentHandler(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(ISO_8859_1)), linkContentHandler, metadata, new ParseContext()); assertEquals("", linkContentHandler.getLinks().get(0).getText()); Schema schema = new HTMLSchema(); schema.elementType("a", HTMLSchema.M_ANY, 65535, 0); ParseContext parseContext = new ParseContext(); parseContext.set(Schema.class, schema); linkContentHandler = new LinkContentHandler(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(ISO_8859_1)), linkContentHandler, metadata, parseContext); assertEquals("\ttext\n\n", linkContentHandler.getLinks().get(0).getText()); }
@Test public void testLocator() throws Exception { final int line = 0; final int col = 1; final int[] textPosition = new int[2]; new HtmlParser().parse(HtmlParserTest.class.getResourceAsStream("/test-documents/testHTML.html"), new ContentHandler() { Locator locator; public void setDocumentLocator(Locator locator) { this.locator = locator; } public void startDocument() throws SAXException { } public void endDocument() throws SAXException { } public void startPrefixMapping(String prefix, String uri) throws SAXException { } public void endPrefixMapping(String prefix) throws SAXException { } public void startElement(String uri, String localName, String qName, Attributes atts) throws SAXException { } public void endElement(String uri, String localName, String qName) throws SAXException { } public void characters(char[] ch, int start, int length) throws SAXException { String text = new String(ch, start, length); if (text.equals("Test Indexation Html") && locator != null) { textPosition[line] = locator.getLineNumber(); textPosition[col] = locator.getColumnNumber(); } } public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { } public void processingInstruction(String target, String data) throws SAXException { } public void skippedEntity(String name) throws SAXException { } }, new Metadata(), new ParseContext()); assertEquals(24, textPosition[line]); assertTrue(Math.abs(textPosition[col] - 47) < 10); }
@Test public void testFirstTitleValueisSetToMetadata() throws Exception { String test = "<html><title>Simple Content</title><body><h1></h1>" + "<title>TitleToIgnore</title></body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("Simple Content", metadata.get(TikaCoreProperties.TITLE)); }
@Test public void testMisleadingMetaContentTypeTags() throws Exception { String test = "<html><head><meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-ELEVEN\">" + "</head><title>title</title><body>body</body></html>"; Metadata metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("text/html; charset=UTF-ELEVEN", metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT)); assertEquals("text/html; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); test = "<html><head><meta http-equiv=\"content-type\" content=\"application/pdf\">" + "</head><title>title</title><body>body</body></html>"; metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("application/pdf", metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT)); assertEquals("text/html; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); test = "<html><head><meta http-equiv=\"content-type\" content=\"application/pdf\" content=\"application/ms-word\">" + "</head><title>title</title><body>body</body></html>"; metadata = new Metadata(); new HtmlParser().parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("application/ms-word", metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT)); assertEquals("text/html; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); }
@Test public void testXHTMLWithMisleading() throws Exception { String test = "<?xml version=\"1.0\" ?>" + "<!DOCTYPE html PUBLIC \"- "<html xmlns=\"http: "<head>\n" + "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\" />\n" + "<title>title</title></head><body>body</body></html>"; Metadata metadata = new Metadata(); AUTO_DETECT_PARSER.parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("text/html; charset=iso-8859-1", metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT)); assertEquals("application/xhtml+xml; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); test = "<?xml version=\"1.0\" ?>" + "<!DOCTYPE html PUBLIC \"- "<html xmlns=\"http: "<head>\n" + "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-NUMBER_SEVEN\" />\n" + "<title>title</title></head><body>body</body></html>"; metadata = new Metadata(); AUTO_DETECT_PARSER.parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("text/html; charset=iso-NUMBER_SEVEN", metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT)); assertEquals("application/xhtml+xml; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); }
@Test public void testAllHeadElements() throws Exception { ParseContext context = new ParseContext(); context.set(HtmlMapper.class, IdentityHtmlMapper.INSTANCE); Metadata metadata = new Metadata(); metadata.set(Metadata.CONTENT_TYPE, "text/html"); final Map<String, Integer> tagFrequencies = new HashMap<>(); String path = "/test-documents/testHTML_head.html"; try (InputStream stream = HtmlParserTest.class.getResourceAsStream(path)) { ContentHandler tagCounter = new DefaultHandler() { @Override public void startElement( String uri, String local, String name, Attributes attributes) throws SAXException { int count = tagFrequencies.containsKey(name) ? tagFrequencies.get(name) : 0; tagFrequencies.put(name, count + 1); } }; new HtmlParser().parse(stream, tagCounter, metadata, context); } assertEquals(1, (int)tagFrequencies.get("title")); assertEquals(9, (int)tagFrequencies.get("meta")); assertEquals(12, (int)tagFrequencies.get("link")); assertEquals(6, (int)tagFrequencies.get("script")); }
@Test public void testParseAscii() throws Exception { String path = "/test-documents/testHTML.html"; final StringWriter href = new StringWriter(); final StringWriter name = new StringWriter(); ContentHandler body = new BodyContentHandler(); Metadata metadata = new Metadata(); try (InputStream stream = HtmlParserTest.class.getResourceAsStream(path)) { ContentHandler link = new DefaultHandler() { @Override public void startElement( String u, String l, String n, Attributes a) throws SAXException { if ("a".equals(l)) { if (a.getValue("href") != null) { href.append(a.getValue("href")); } else if (a.getValue("name") != null) { name.append(a.getValue("name")); } } } }; new HtmlParser().parse( stream, new TeeContentHandler(body, link), metadata, new ParseContext()); } assertEquals( "Title : Test Indexation Html", metadata.get(TikaCoreProperties.TITLE)); assertEquals("Tika Developers", metadata.get("Author")); assertEquals("5", metadata.get("refresh")); assertEquals("51.2312", metadata.get(Geographic.LATITUDE)); assertEquals("-5.1987", metadata.get(Geographic.LONGITUDE)); assertEquals("http: assertEquals("test-anchor", name.toString()); String content = body.toString(); assertTrue( "Did not contain expected text:" + "Test Indexation Html", content.contains("Test Indexation Html")); assertTrue( "Did not contain expected text:" + "Indexation du fichier", content.contains("Indexation du fichier")); }
@Test public void testParseEmpty() throws Exception { ContentHandler handler = new BodyContentHandler(); new HtmlParser().parse( new ByteArrayInputStream(new byte[0]), handler, new Metadata(), new ParseContext()); assertEquals("", handler.toString()); }