JsoupTextExtractor implements TextExtractor { @Override public ParsedContent extractContent(InputStream inputStream, ContentType contentType) throws Exception { if (inputStream == null || contentType == null) { return ParsedContent.empty(); } Charset charset = contentType.charset().orElse(StandardCharsets.UTF_8); if (contentType.mimeType().equals(TEXT_HTML)) { return parseHtmlContent(inputStream, charset); } if (contentType.mimeType().equals(TEXT_PLAIN)) { return parsePlainTextContent(inputStream, charset); } return ParsedContent.empty(); }  @Override ParsedContent extractContent(InputStream inputStream, ContentType contentType);  }
@Test void extractedTextFromHtmlShouldNotContainTheContentOfTitleTag() throws Exception { InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/html.txt"); assertThat(textExtractor.extractContent(inputStream, TEXT_HTML_CONTENT_TYPE).getTextualContent().get()) .doesNotContain("*|MC:SUBJECT|*"); }
@Test void extractContentShouldHandlePlainText() throws Exception { InputStream inputStream = new ByteArrayInputStream("myText".getBytes(StandardCharsets.UTF_8)); assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain")).getTextualContent()) .contains("myText"); }
@Test void extractContentShouldHandlePlainTextWithCharset() throws Exception { InputStream inputStream = new ByteArrayInputStream("myText".getBytes(StandardCharsets.UTF_8)); assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain; charset=utf-8")).getTextualContent()) .contains("myText"); }
@Test void extractContentShouldHandleArbitraryTextMediaType() throws Exception { InputStream inputStream = new ByteArrayInputStream("myText".getBytes(StandardCharsets.UTF_8)); assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/arbitrary")).getTextualContent()) .isEmpty(); }
@Test void extractContentShouldReturnEmptyWhenNullData() throws Exception { assertThat(textExtractor.extractContent(null, TEXT_HTML_CONTENT_TYPE)) .isEqualTo(ParsedContent.empty()); }
@Test void extractContentShouldReturnEmptyWhenNullContentType() throws Exception { InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/html.txt"); assertThat(textExtractor.extractContent(inputStream, null)) .isEqualTo(ParsedContent.empty()); }
@Test void extractContentShouldNotThrowWhenContainingNullCharacters() throws Exception { InputStream inputStream = textContentWithManyNullCharacters(); assertThat(textExtractor.extractContent(inputStream, TEXT_HTML_CONTENT_TYPE) .getTextualContent()) .hasValueSatisfying(text -> assertThat(text) .contains(HTML_TEXT_CONTENT)); }
@Test void extractContentShouldTakeIntoAccountCharsetWhenPlainText() throws Exception { InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/simple-text-iso-8859-1.txt"); assertThat(inputStream).isNotNull(); assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/plain; charset=ISO-8859-1")) .getTextualContent()) .contains("\"é\" This text is not UTF-8 \"à\""); }
@Test void extractContentTakeIntoAccountCharsetWhenHTML() throws Exception { InputStream inputStream = ClassLoader.getSystemResourceAsStream("documents/html-iso-8859-1.html"); assertThat(inputStream).isNotNull(); assertThat(textExtractor.extractContent(inputStream, ContentType.of("text/html; charset=ISO-8859-1")) .getTextualContent()) .contains("\"é\" this is a simple HTML text \"à\""); }