HtmlDocumentReader extends JCasResourceCollectionReader_ImplBase { @Override public void getNext(JCas aJCas) throws IOException, CollectionException { Resource res = nextFile(); initCas(aJCas, res); CAS cas = aJCas.getCas(); String html; try (InputStream is = new BufferedInputStream( CompressionUtils.getInputStream(res.getLocation(), res.getInputStream()))) { if (ENCODING_AUTO.equals(sourceEncoding)) { CharsetDetector detector = new CharsetDetector(); html = IOUtils.toString(detector.getReader(is, null)); } else { html = IOUtils.toString(is, sourceEncoding); } } Document doc = Jsoup.parse(html); CasXmlHandler handler = new CasXmlHandler(aJCas); NodeTraversor traversor = new NodeTraversor(new NodeVisitor() { @Override public void head(Node node, int depth) { try { if (node instanceof Document) { handler.startDocument(); handler.captureText(false); } else if (node instanceof TextNode) { TextNode textNode = (TextNode) node; StringBuilder buffer = new StringBuilder(); if (normalizeWhitespace) { appendNormalisedText(buffer, textNode); } else { buffer.append(textNode.getWholeText()); } char[] text = buffer.toString().toCharArray(); handler.characters(text, 0, text.length); } else if (node instanceof Element) { Element element = (Element) node; if ( handler.getText().length() > 0 && (element.isBlock() || element.nodeName().equals("br")) && !lastCharIsWhitespace(handler.getText()) ) { char[] text = " ".toCharArray(); handler.characters(text, 0, text.length); } AttributesImpl attributes = new AttributesImpl(); if (element.attributes() != null) { for (Attribute attr : element.attributes()) { attributes.addAttribute("", "", attr.getKey(), "CDATA", attr.getValue()); } } if ("body".equals(element.tagName())) { handler.captureText(true); } handler.startElement("", "", element.tagName(), attributes); } } catch (Exception e) { throw new RuntimeException(e); } } @Override public void tail(Node node, int depth) { try { if (node instanceof Document) { handler.endDocument(); } else if (node instanceof Element) { Element element = (Element) node; XmlElement elementFS = handler.getCurrentElement(); handler.endElement("", "", element.tagName()); if ("body".equals(element.tagName())) { handler.captureText(false); } Integer type = mappings.get(node.nodeName()); if (type != null) { int[] span = { elementFS.getBegin(), elementFS.getEnd() }; trim(handler.getText(), span); Div div = (Div) cas.createAnnotation(aJCas.getCasType(type), span[0], span[1]); div.setDivType(node.nodeName()); div.addToIndexes(); } } } catch (Exception e) { throw new RuntimeException(e); } } }); traversor.traverse(doc); }  @Override void initialize(UimaContext aContext); @Override void getNext(JCas aJCas); static final String ENCODING_AUTO; static final String PARAM_SOURCE_ENCODING; static final String PARAM_NORMALIZE_WHITESPACE; }
@Test public void testReadFileWithOnlyBody() throws Exception { JCas jcas = JCasFactory.createJCas(); CollectionReader reader = createReader(HtmlDocumentReader.class, HtmlDocumentReader.PARAM_SOURCE_LOCATION, "src/test/resources/html/test.html", HtmlDocumentReader.PARAM_LANGUAGE, "en"); reader.getNext(jcas.getCas()); assertThat(jcas.getDocumentText()) .isEqualTo(" Heading This is the first paragraph. This is the second paragraph. "); assertThat(select(jcas, Heading.class)) .extracting(Heading::getCoveredText) .containsExactly("Heading"); assertThat(select(jcas, Paragraph.class)) .extracting(Paragraph::getCoveredText) .containsExactly("This is the first paragraph.", "This is the second paragraph."); }
@Test public void testReadFileWithHead() throws Exception { JCas jcas = JCasFactory.createJCas(); CollectionReader reader = createReader(HtmlDocumentReader.class, HtmlDocumentReader.PARAM_SOURCE_LOCATION, "src/test/resources/html/test-with-head.html", HtmlDocumentReader.PARAM_LANGUAGE, "en"); reader.getNext(jcas.getCas()); assertThat(jcas.getDocumentText()) .isEqualTo(" Heading This is the first paragraph. This is the second paragraph. "); assertThat(select(jcas, Heading.class)) .extracting(Heading::getCoveredText) .containsExactly("Heading"); assertThat(select(jcas, Paragraph.class)) .extracting(Paragraph::getCoveredText) .containsExactly("This is the first paragraph.", "This is the second paragraph."); }