@Test public void testQuoted() throws Exception { ContentHandler handler = new BodyContentHandler(); Metadata metadata = new Metadata(); try (InputStream stream = getResourceAsStream("/test-documents/quoted.mbox")) { mboxParser.parse(stream, handler, metadata, recursingContext); } assertContains("Test content", handler.toString()); assertContains("> quoted stuff", handler.toString()); }
public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException { EmbeddedDocumentExtractor extractor = EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); String charsetName = "windows-1252"; metadata.set(Metadata.CONTENT_TYPE, MBOX_MIME_TYPE); metadata.set(Metadata.CONTENT_ENCODING, charsetName); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); InputStreamReader isr = new InputStreamReader(stream, charsetName); try (BufferedReader reader = new BufferedReader(isr)) { String curLine = reader.readLine(); int mailItem = 0; do { if (curLine.startsWith(MBOX_RECORD_DIVIDER)) { Metadata mailMetadata = new Metadata(); Queue<String> multiline = new LinkedList<String>(); mailMetadata.add(EMAIL_FROMLINE_METADATA, curLine.substring(MBOX_RECORD_DIVIDER.length())); mailMetadata.set(Metadata.CONTENT_TYPE, "message/rfc822"); mailMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, "message/rfc822"); curLine = reader.readLine(); if (curLine == null) { break; } ByteArrayOutputStream message = new ByteArrayOutputStream(100000); do { if (curLine.startsWith(" ") || curLine.startsWith("\t")) { String latestLine = multiline.poll(); latestLine += " " + curLine.trim(); multiline.add(latestLine); } else { multiline.add(curLine); } message.write(curLine.getBytes(charsetName)); message.write(0x0A); curLine = reader.readLine(); } while (curLine != null && !curLine.startsWith(MBOX_RECORD_DIVIDER) && message.size() < MAIL_MAX_SIZE); for (String item : multiline) { saveHeaderInMetadata(mailMetadata, item); } ByteArrayInputStream messageStream = new ByteArrayInputStream(message.toByteArray()); message = null; if (extractor.shouldParseEmbedded(mailMetadata)) { extractor.parseEmbedded(messageStream, xhtml, mailMetadata, true); } if (tracking) { getTrackingMetadata().put(mailItem++, mailMetadata); } } else { curLine = reader.readLine(); } } while (curLine != null && !Thread.currentThread().isInterrupted()); } xhtml.endDocument(); }
MboxParser extends AbstractParser { public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException { EmbeddedDocumentExtractor extractor = EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); String charsetName = "windows-1252"; metadata.set(Metadata.CONTENT_TYPE, MBOX_MIME_TYPE); metadata.set(Metadata.CONTENT_ENCODING, charsetName); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); InputStreamReader isr = new InputStreamReader(stream, charsetName); try (BufferedReader reader = new BufferedReader(isr)) { String curLine = reader.readLine(); int mailItem = 0; do { if (curLine.startsWith(MBOX_RECORD_DIVIDER)) { Metadata mailMetadata = new Metadata(); Queue<String> multiline = new LinkedList<String>(); mailMetadata.add(EMAIL_FROMLINE_METADATA, curLine.substring(MBOX_RECORD_DIVIDER.length())); mailMetadata.set(Metadata.CONTENT_TYPE, "message/rfc822"); mailMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, "message/rfc822"); curLine = reader.readLine(); if (curLine == null) { break; } ByteArrayOutputStream message = new ByteArrayOutputStream(100000); do { if (curLine.startsWith(" ") || curLine.startsWith("\t")) { String latestLine = multiline.poll(); latestLine += " " + curLine.trim(); multiline.add(latestLine); } else { multiline.add(curLine); } message.write(curLine.getBytes(charsetName)); message.write(0x0A); curLine = reader.readLine(); } while (curLine != null && !curLine.startsWith(MBOX_RECORD_DIVIDER) && message.size() < MAIL_MAX_SIZE); for (String item : multiline) { saveHeaderInMetadata(mailMetadata, item); } ByteArrayInputStream messageStream = new ByteArrayInputStream(message.toByteArray()); message = null; if (extractor.shouldParseEmbedded(mailMetadata)) { extractor.parseEmbedded(messageStream, xhtml, mailMetadata, true); } if (tracking) { getTrackingMetadata().put(mailItem++, mailMetadata); } } else { curLine = reader.readLine(); } } while (curLine != null && !Thread.currentThread().isInterrupted()); } xhtml.endDocument(); } }
MboxParser extends AbstractParser { public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException { EmbeddedDocumentExtractor extractor = EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); String charsetName = "windows-1252"; metadata.set(Metadata.CONTENT_TYPE, MBOX_MIME_TYPE); metadata.set(Metadata.CONTENT_ENCODING, charsetName); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); InputStreamReader isr = new InputStreamReader(stream, charsetName); try (BufferedReader reader = new BufferedReader(isr)) { String curLine = reader.readLine(); int mailItem = 0; do { if (curLine.startsWith(MBOX_RECORD_DIVIDER)) { Metadata mailMetadata = new Metadata(); Queue<String> multiline = new LinkedList<String>(); mailMetadata.add(EMAIL_FROMLINE_METADATA, curLine.substring(MBOX_RECORD_DIVIDER.length())); mailMetadata.set(Metadata.CONTENT_TYPE, "message/rfc822"); mailMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, "message/rfc822"); curLine = reader.readLine(); if (curLine == null) { break; } ByteArrayOutputStream message = new ByteArrayOutputStream(100000); do { if (curLine.startsWith(" ") || curLine.startsWith("\t")) { String latestLine = multiline.poll(); latestLine += " " + curLine.trim(); multiline.add(latestLine); } else { multiline.add(curLine); } message.write(curLine.getBytes(charsetName)); message.write(0x0A); curLine = reader.readLine(); } while (curLine != null && !curLine.startsWith(MBOX_RECORD_DIVIDER) && message.size() < MAIL_MAX_SIZE); for (String item : multiline) { saveHeaderInMetadata(mailMetadata, item); } ByteArrayInputStream messageStream = new ByteArrayInputStream(message.toByteArray()); message = null; if (extractor.shouldParseEmbedded(mailMetadata)) { extractor.parseEmbedded(messageStream, xhtml, mailMetadata, true); } if (tracking) { getTrackingMetadata().put(mailItem++, mailMetadata); } } else { curLine = reader.readLine(); } } while (curLine != null && !Thread.currentThread().isInterrupted()); } xhtml.endDocument(); }  }
MboxParser extends AbstractParser { public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException { EmbeddedDocumentExtractor extractor = EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); String charsetName = "windows-1252"; metadata.set(Metadata.CONTENT_TYPE, MBOX_MIME_TYPE); metadata.set(Metadata.CONTENT_ENCODING, charsetName); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); InputStreamReader isr = new InputStreamReader(stream, charsetName); try (BufferedReader reader = new BufferedReader(isr)) { String curLine = reader.readLine(); int mailItem = 0; do { if (curLine.startsWith(MBOX_RECORD_DIVIDER)) { Metadata mailMetadata = new Metadata(); Queue<String> multiline = new LinkedList<String>(); mailMetadata.add(EMAIL_FROMLINE_METADATA, curLine.substring(MBOX_RECORD_DIVIDER.length())); mailMetadata.set(Metadata.CONTENT_TYPE, "message/rfc822"); mailMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, "message/rfc822"); curLine = reader.readLine(); if (curLine == null) { break; } ByteArrayOutputStream message = new ByteArrayOutputStream(100000); do { if (curLine.startsWith(" ") || curLine.startsWith("\t")) { String latestLine = multiline.poll(); latestLine += " " + curLine.trim(); multiline.add(latestLine); } else { multiline.add(curLine); } message.write(curLine.getBytes(charsetName)); message.write(0x0A); curLine = reader.readLine(); } while (curLine != null && !curLine.startsWith(MBOX_RECORD_DIVIDER) && message.size() < MAIL_MAX_SIZE); for (String item : multiline) { saveHeaderInMetadata(mailMetadata, item); } ByteArrayInputStream messageStream = new ByteArrayInputStream(message.toByteArray()); message = null; if (extractor.shouldParseEmbedded(mailMetadata)) { extractor.parseEmbedded(messageStream, xhtml, mailMetadata, true); } if (tracking) { getTrackingMetadata().put(mailItem++, mailMetadata); } } else { curLine = reader.readLine(); } } while (curLine != null && !Thread.currentThread().isInterrupted()); } xhtml.endDocument(); }  Set<MediaType> getSupportedTypes(ParseContext context); void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context); boolean isTracking(); void setTracking(boolean tracking); Map<Integer, Metadata> getTrackingMetadata(); }
MboxParser extends AbstractParser { public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException { EmbeddedDocumentExtractor extractor = EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); String charsetName = "windows-1252"; metadata.set(Metadata.CONTENT_TYPE, MBOX_MIME_TYPE); metadata.set(Metadata.CONTENT_ENCODING, charsetName); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); InputStreamReader isr = new InputStreamReader(stream, charsetName); try (BufferedReader reader = new BufferedReader(isr)) { String curLine = reader.readLine(); int mailItem = 0; do { if (curLine.startsWith(MBOX_RECORD_DIVIDER)) { Metadata mailMetadata = new Metadata(); Queue<String> multiline = new LinkedList<String>(); mailMetadata.add(EMAIL_FROMLINE_METADATA, curLine.substring(MBOX_RECORD_DIVIDER.length())); mailMetadata.set(Metadata.CONTENT_TYPE, "message/rfc822"); mailMetadata.set(TikaCoreProperties.CONTENT_TYPE_OVERRIDE, "message/rfc822"); curLine = reader.readLine(); if (curLine == null) { break; } ByteArrayOutputStream message = new ByteArrayOutputStream(100000); do { if (curLine.startsWith(" ") || curLine.startsWith("\t")) { String latestLine = multiline.poll(); latestLine += " " + curLine.trim(); multiline.add(latestLine); } else { multiline.add(curLine); } message.write(curLine.getBytes(charsetName)); message.write(0x0A); curLine = reader.readLine(); } while (curLine != null && !curLine.startsWith(MBOX_RECORD_DIVIDER) && message.size() < MAIL_MAX_SIZE); for (String item : multiline) { saveHeaderInMetadata(mailMetadata, item); } ByteArrayInputStream messageStream = new ByteArrayInputStream(message.toByteArray()); message = null; if (extractor.shouldParseEmbedded(mailMetadata)) { extractor.parseEmbedded(messageStream, xhtml, mailMetadata, true); } if (tracking) { getTrackingMetadata().put(mailItem++, mailMetadata); } } else { curLine = reader.readLine(); } } while (curLine != null && !Thread.currentThread().isInterrupted()); } xhtml.endDocument(); }  Set<MediaType> getSupportedTypes(ParseContext context); void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context); boolean isTracking(); void setTracking(boolean tracking); Map<Integer, Metadata> getTrackingMetadata(); static final String MBOX_MIME_TYPE; static final String MBOX_RECORD_DIVIDER; static final int MAIL_MAX_SIZE; }